From ed5fd919ef68ac29bafbd2fbdc436226f40448d3 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 3 Jun 2021 18:51:00 -0700 Subject: [PATCH] Update dockerfiles to use the latest cmake (#7933) --- dockerfiles/Dockerfile.arm32v7 | 8 ++++---- dockerfiles/Dockerfile.cuda | 10 +++++----- dockerfiles/Dockerfile.migraphx | 4 ++-- dockerfiles/Dockerfile.openvino | 2 +- dockerfiles/Dockerfile.openvino-csharp | 2 +- dockerfiles/Dockerfile.rocm | 4 ++-- dockerfiles/Dockerfile.source | 6 +++--- dockerfiles/Dockerfile.tensorrt | 4 ++-- dockerfiles/Dockerfile.training | 17 +++++++++-------- dockerfiles/Dockerfile.vitisai | 4 ++-- dockerfiles/scripts/install_common_deps.sh | 6 +++--- .../set-python-manylinux-variables-step.yml | 19 +++++-------------- 12 files changed, 39 insertions(+), 47 deletions(-) diff --git a/dockerfiles/Dockerfile.arm32v7 b/dockerfiles/Dockerfile.arm32v7 index bcc7fc55f7..f0cc9a599f 100644 --- a/dockerfiles/Dockerfile.arm32v7 +++ b/dockerfiles/Dockerfile.arm32v7 @@ -31,10 +31,10 @@ RUN pip3 install numpy # Build the latest cmake WORKDIR /code -RUN wget https://github.com/Kitware/CMake/releases/download/v3.18.3/cmake-3.18.3.tar.gz -RUN tar zxf cmake-3.18.3.tar.gz +RUN wget https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3.tar.gz +RUN tar zxf cmake-3.20.3.tar.gz -WORKDIR /code/cmake-3.18.3 +WORKDIR /code/cmake-3.20.3 RUN ./configure --system-curl RUN make RUN sudo make install @@ -50,7 +50,7 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_SERVER_BRANCH} --recursive # Build ORT including the shared lib and python bindings WORKDIR /code/onnxruntime -RUN ./build.sh --use_openmp ${BUILDARGS} --update --build --build_shared_lib --build_wheel +RUN ./build.sh ${BUILDARGS} --update --build --build_shared_lib --build_wheel # Build Output RUN ls -l /code/onnxruntime/build/Linux/${BUILDTYPE}/*.so diff --git a/dockerfiles/Dockerfile.cuda b/dockerfiles/Dockerfile.cuda index 791a02e4cc..3cb38b2875 100644 --- a/dockerfiles/Dockerfile.cuda +++ b/dockerfiles/Dockerfile.cuda @@ -4,18 +4,18 @@ # -------------------------------------------------------------- # Dockerfile to run ONNXRuntime with CUDA, CUDNN integration -# nVidia cuda 10.2 Base Image -FROM nvidia/cuda:10.2-cudnn8-devel +# nVidia cuda 11.1 Base Image +FROM nvcr.io/nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04 MAINTAINER Changming Sun "chasun@microsoft.com" ADD . /code ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.18.2-Linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.18.2-Linux-x86_64.tar.gz --strip=1 -C /usr +RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.20.3-Linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr RUN cd /code && /bin/bash ./build.sh --skip_submodule_sync --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_cuda --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) 'CMAKE_CUDA_ARCHITECTURES=30;37;50;52;60;70' -FROM nvidia/cuda:10.2-cudnn8-runtime +FROM nvcr.io/nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu18.04 COPY --from=0 /code/build/Linux/Release/dist /root COPY --from=0 /code/dockerfiles/LICENSE-IMAGE.txt /code/LICENSE-IMAGE.txt ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 ca-certificates python3-setuptools python3-wheel python3-pip unattended-upgrades && unattended-upgrade && python3 -m pip install /root/*.whl && rm -rf /root/*.whl \ No newline at end of file +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 ca-certificates python3-setuptools python3-wheel python3-pip unattended-upgrades && unattended-upgrade && python3 -m pip install /root/*.whl && rm -rf /root/*.whl diff --git a/dockerfiles/Dockerfile.migraphx b/dockerfiles/Dockerfile.migraphx index 3b88714a3d..4cd8f6d9db 100644 --- a/dockerfiles/Dockerfile.migraphx +++ b/dockerfiles/Dockerfile.migraphx @@ -29,7 +29,7 @@ RUN apt-get update &&\ # Install rbuild RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz -ENV PATH /opt/miniconda/bin:/code/cmake-3.14.3-Linux-x86_64/bin:${PATH} +ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-Linux-x86_64/bin:${PATH} # Install MIGraphX from source RUN mkdir -p /migraphx @@ -47,4 +47,4 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR /bin/sh ./build.sh --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --use_migraphx &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.14.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino index 1d8771903f..e2580797af 100644 --- a/dockerfiles/Dockerfile.openvino +++ b/dockerfiles/Dockerfile.openvino @@ -12,7 +12,7 @@ ARG ONNXRUNTIME_BRANCH=master WORKDIR /code ARG MY_ROOT=/code -ENV PATH /opt/miniconda/bin:/code/cmake-3.14.3-Linux-x86_64/bin:$PATH +ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-Linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH=/opt/miniconda/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.3.394 diff --git a/dockerfiles/Dockerfile.openvino-csharp b/dockerfiles/Dockerfile.openvino-csharp index e79214a3d4..32d627d07c 100644 --- a/dockerfiles/Dockerfile.openvino-csharp +++ b/dockerfiles/Dockerfile.openvino-csharp @@ -12,7 +12,7 @@ ARG ONNXRUNTIME_BRANCH=master WORKDIR /code ARG MY_ROOT=/code -ENV PATH /opt/miniconda/bin:/code/cmake-3.14.3-Linux-x86_64/bin:$PATH +ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-Linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH=/opt/miniconda/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.3.394 diff --git a/dockerfiles/Dockerfile.rocm b/dockerfiles/Dockerfile.rocm index 1fdc467180..9805bd69ea 100644 --- a/dockerfiles/Dockerfile.rocm +++ b/dockerfiles/Dockerfile.rocm @@ -29,7 +29,7 @@ RUN apt-get update &&\ # Install yapf RUN pip3 install yapf==0.28.0 -ENV PATH /opt/miniconda/bin:/code/cmake-3.14.3-Linux-x86_64/bin:${PATH} +ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-Linux-x86_64/bin:${PATH} # Install dependencies COPY ./scripts/install_rocm_deps.sh / @@ -45,5 +45,5 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --use_rocm --rocm_home=/opt/rocm &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.14.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 diff --git a/dockerfiles/Dockerfile.source b/dockerfiles/Dockerfile.source index 0c83907195..eb5ceb82ae 100644 --- a/dockerfiles/Dockerfile.source +++ b/dockerfiles/Dockerfile.source @@ -9,12 +9,12 @@ MAINTAINER Changming Sun "chasun@microsoft.com" ADD . /code ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.18.2-Linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.18.2-Linux-x86_64.tar.gz --strip=1 -C /usr +RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.20.3-Linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr # Prepare onnxruntime repository & build onnxruntime -RUN cd /code && /bin/bash ./build.sh --skip_submodule_sync --use_openmp --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) +RUN cd /code && /bin/bash ./build.sh --skip_submodule_sync --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) FROM ubuntu:18.04 COPY --from=0 /code/build/Linux/Release/dist /root COPY --from=0 /code/dockerfiles/LICENSE-IMAGE.txt /code/LICENSE-IMAGE.txt -RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgomp1 ca-certificates python3-setuptools python3-wheel python3-pip unattended-upgrades && unattended-upgrade && python3 -m pip install /root/*.whl && rm -rf /root/*.whl \ No newline at end of file +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 ca-certificates python3-setuptools python3-wheel python3-pip unattended-upgrades && unattended-upgrade && python3 -m pip install /root/*.whl && rm -rf /root/*.whl \ No newline at end of file diff --git a/dockerfiles/Dockerfile.tensorrt b/dockerfiles/Dockerfile.tensorrt index 87e088e414..872c3e29af 100644 --- a/dockerfiles/Dockerfile.tensorrt +++ b/dockerfiles/Dockerfile.tensorrt @@ -15,7 +15,7 @@ RUN apt-get update &&\ RUN unattended-upgrade WORKDIR /code -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/code/cmake-3.14.3-Linux-x86_64/bin:/opt/miniconda/bin:${PATH} +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/code/cmake-3.20.3-Linux-x86_64/bin:/opt/miniconda/bin:${PATH} # Prepare onnxruntime repository & build onnxruntime with TensorRT RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime &&\ @@ -27,4 +27,4 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR /bin/sh ./build.sh --parallel --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /workspace/tensorrt --config Release --build_wheel --update --build --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.14.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 diff --git a/dockerfiles/Dockerfile.training b/dockerfiles/Dockerfile.training index 6444b18e1e..6c8c78a397 100644 --- a/dockerfiles/Dockerfile.training +++ b/dockerfiles/Dockerfile.training @@ -14,7 +14,7 @@ ARG OPENMPI_PATH=/opt/openmpi-${OPENMPI_VERSION} ARG COMMIT=master # cuda development image for building sources -FROM nvidia/cuda:10.2-cudnn8-devel-ubuntu18.04 as builder +FROM nvcr.io/nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04 as builder # set location for builds WORKDIR /stage @@ -32,7 +32,10 @@ RUN apt-get -y update &&\ RUN unattended-upgrade RUN locale-gen en_US.UTF-8 && \ - update-locale LANG=en_US.UTF-8 + update-locale LANG=en_US.UTF-8 && \ + curl -O -L https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz && \ + tar -zxf cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr && \ + rm -rf cmake-3.20.3-Linux-x86_64.tar.gz # install miniconda (comes with python 3.7 default) ARG CONDA_VERSION @@ -42,12 +45,11 @@ RUN cd /stage && curl -fSsL --insecure ${CONDA_URL} -o install-conda.sh &&\ /opt/conda/bin/conda clean -ya ENV PATH=/opt/conda/bin:${PATH} -# install cmake, setuptools, numpy, and onnx +# install setuptools, numpy, and onnx ARG NUMPY_VERSION ARG ONNX_VERSION RUN conda install -y \ setuptools \ - cmake \ numpy=${NUMPY_VERSION} &&\ pip install \ onnx=="${ONNX_VERSION}" @@ -147,7 +149,7 @@ RUN cd /stage && git clone https://github.com/microsoft/onnxruntime.git &&\ --build_dir build \ --build \ --build_wheel \ - --skip_tests &&\ + --skip_tests --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=35;37;50;52;60;61;70;75;80;86' &&\ pip install build/${BUILD_CONFIG}/dist/*.whl # Install AzureML support and commonly used packages. @@ -155,7 +157,7 @@ RUN pip install azureml-defaults sentencepiece==0.1.92 transformers==2.11.0 msgp # switch to cuda runtime environment # note: launch with --gpus all or nvidia-docker -FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 +FROM nvcr.io/nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu18.04 WORKDIR /stage # install ucx @@ -215,8 +217,7 @@ LABEL PYTORCH_VERSION=${PYTORCH_VERSION} # clean\finalize environment # note: adds onnxruntime license and third party notices -RUN conda remove -y cmake &&\ - apt-get purge -y build-essential &&\ +RUN apt-get purge -y build-essential &&\ apt-get autoremove -y &&\ rm -fr /stage WORKDIR /workspace diff --git a/dockerfiles/Dockerfile.vitisai b/dockerfiles/Dockerfile.vitisai index ccff3fe1c6..4baf77d723 100644 --- a/dockerfiles/Dockerfile.vitisai +++ b/dockerfiles/Dockerfile.vitisai @@ -22,7 +22,7 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -ENV PATH /code/cmake-3.14.3-Linux-x86_64/bin:$PATH +ENV PATH /code/cmake-3.20.3-Linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH /opt/xilinx/xrt/lib:$LD_LIBRARY_PATH WORKDIR /code @@ -41,4 +41,4 @@ RUN . $VAI_ROOT/conda/etc/profile.d/conda.sh &&\ /bin/sh ./build.sh --use_openmp --config RelWithDebInfo --enable_pybind --build_wheel --use_vitisai --parallel --update --build --build_shared_lib &&\ pip install /code/onnxruntime/build/Linux/RelWithDebInfo/dist/*-linux_x86_64.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.14.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 diff --git a/dockerfiles/scripts/install_common_deps.sh b/dockerfiles/scripts/install_common_deps.sh index 26d246ee5a..e32d7a4632 100644 --- a/dockerfiles/scripts/install_common_deps.sh +++ b/dockerfiles/scripts/install_common_deps.sh @@ -19,6 +19,6 @@ pip install numpy rm -rf /opt/miniconda/pkgs # Dependencies: cmake -wget --quiet https://github.com/Kitware/CMake/releases/download/v3.14.3/cmake-3.14.3-Linux-x86_64.tar.gz -tar zxf cmake-3.14.3-Linux-x86_64.tar.gz -rm -rf cmake-3.14.3-Linux-x86_64.tar.gz +wget --quiet https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz +tar zxf cmake-3.20.3-Linux-x86_64.tar.gz +rm -rf cmake-3.20.3-Linux-x86_64.tar.gz diff --git a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml index 458e0e4887..8c3adbfde0 100644 --- a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml +++ b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml @@ -9,30 +9,21 @@ steps: script: | version = "$(PythonVersion)" - if version == "3.5": + if version == "3.6": variables = { - "PythonManylinuxDir": "/opt/python/cp35-cp35m", - "PythonManylinuxIncludeDir": "/opt/python/cp35-cp35m/include/python3.5m", - } - elif version == "3.6": - variables = { - "PythonManylinuxDir": "/opt/python/cp36-cp36m", - "PythonManylinuxIncludeDir": "/opt/python/cp36-cp36m/include/python3.6m", + "PythonManylinuxDir": "/opt/python/cp36-cp36m" } elif version == "3.7": variables = { - "PythonManylinuxDir": "/opt/python/cp37-cp37m", - "PythonManylinuxIncludeDir": "/opt/python/cp37-cp37m/include/python3.7m", + "PythonManylinuxDir": "/opt/python/cp37-cp37m" } elif version == "3.8": variables = { - "PythonManylinuxDir": "/opt/python/cp38-cp38", - "PythonManylinuxIncludeDir": "/opt/python/cp38-cp38/include/python3.8", + "PythonManylinuxDir": "/opt/python/cp38-cp38" } elif version == "3.9": variables = { - "PythonManylinuxDir": "/opt/python/cp39-cp39", - "PythonManylinuxIncludeDir": "/opt/python/cp39-cp39/include/python3.9", + "PythonManylinuxDir": "/opt/python/cp39-cp39" } else: raise ValueError("Unsupported Python version: '{}'".format(version))