Fix nightly CI pipeline to generate ROCm 4.2 wheels and add ROCm 4.3.1 wheels (#9101)

* make work for both rocm 4.2 and rocm 4.3.1 * fix rocm 4.3.1 docker image reference * fix CUDA_VERSION to ROCM_VERSION * fix ReduceConsts conflict def * add ifdef to miopen_common.h as well * trailing ws
2026-07-29 20:14:01 +00:00 · 2021-09-19 23:36:03 -07:00 · 2021-09-19 23:36:03 -07:00 · 47888392ab
commit 47888392ab
parent 23e9c0a7f1
8 changed files with 307 additions and 67 deletions
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -1591,6 +1591,31 @@ if (onnxruntime_USE_ROCM)
  if (onnxruntime_USE_CUDA)
    message(FATAL_ERROR "ROCM does not support build with CUDA!")
  endif()
+
+  if(NOT DEFINED ENV{ROCM_PATH})
+    set(ROCM_PATH /opt/rocm)
+  else()
+    set(ROCM_PATH $ENV{ROCM_PATH})
+  endif()
+
+  # replicate strategy used by pytorch to get ROCM_VERSION
+  # https://github.com/pytorch/pytorch/blob/8eb21488fdcdb8b0e6fa2e46179b5fa6c42e75af/cmake/public/LoadHIP.cmake#L153-L173
+  file(READ "${ROCM_PATH}/.info/version-dev" ROCM_VERSION_DEV_RAW)
+  string(REGEX MATCH "^([0-9]+)\.([0-9]+)\.([0-9]+)-.*$" ROCM_VERSION_DEV_MATCH ${ROCM_VERSION_DEV_RAW})
+  if(ROCM_VERSION_DEV_MATCH)
+    set(ROCM_VERSION_DEV_MAJOR ${CMAKE_MATCH_1})
+    set(ROCM_VERSION_DEV_MINOR ${CMAKE_MATCH_2})
+    set(ROCM_VERSION_DEV_PATCH ${CMAKE_MATCH_3})
+    set(ROCM_VERSION_DEV "${ROCM_VERSION_DEV_MAJOR}.${ROCM_VERSION_DEV_MINOR}.${ROCM_VERSION_DEV_PATCH}")
+    math(EXPR ROCM_VERSION_DEV_INT "(${ROCM_VERSION_DEV_MAJOR}*10000) + (${ROCM_VERSION_DEV_MINOR}*100) + ${ROCM_VERSION_DEV_PATCH}")
+  endif()
+  message("\n***** ROCm version from ${ROCM_PATH}/.info/version-dev ****\n")
+  message("ROCM_VERSION_DEV: ${ROCM_VERSION_DEV}")
+  message("ROCM_VERSION_DEV_MAJOR: ${ROCM_VERSION_DEV_MAJOR}")
+  message("ROCM_VERSION_DEV_MINOR: ${ROCM_VERSION_DEV_MINOR}")
+  message("ROCM_VERSION_DEV_PATCH: ${ROCM_VERSION_DEV_PATCH}")
+  message("ROCM_VERSION_DEV_INT:   ${ROCM_VERSION_DEV_INT}")
+  add_definitions(-DROCM_VERSION=${ROCM_VERSION_DEV_INT})
 endif()

 if (onnxruntime_USE_TVM)
--- a/onnxruntime/core/providers/rocm/miopen_common.cc
+++ b/onnxruntime/core/providers/rocm/miopen_common.cc
@ -88,24 +88,29 @@ const float Consts<half>::Zero = 0;

 const float Consts<half>::One = 1;

-// As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data
+#if ROCM_VERSION >= 40300
+template <>
+const float ReduceConsts<half>::One = 1;
+
+template <>
+const float ReduceConsts<half>::Zero = 0;
+#else
+// Up until ROCm 4.2, miopenReduceTensor() required alpha/beta to be the same data
 // type as the input type. This differs from cudnnReduceTensor() and other
 // MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16).
-//
-// NOTE: this workaround can be removed in ROCm 4.3:
-//       https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914
 template <>
 const half ReduceConsts<half>::One = 1.f;

+template <>
+const half ReduceConsts<half>::Zero = 0.f;
+#endif
+
 template <>
 const float ReduceConsts<float>::One = 1;

 template <>
 const double ReduceConsts<double>::One = 1;

-template <>
-const half ReduceConsts<half>::Zero = 0.f;
-
 template <>
 const float ReduceConsts<float>::Zero = 0;

--- a/onnxruntime/core/providers/rocm/miopen_common.h
+++ b/onnxruntime/core/providers/rocm/miopen_common.h
@ -46,18 +46,23 @@ struct Consts<half> {
  static const float One;
 };

-// As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data
-// type as the input type. This differs from cudnnReduceTensor() and other
-// MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16).
-//
-// NOTE: this workaround can be removed in ROCm 4.3:
-//       https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914
 template <typename ElemType>
 struct ReduceConsts {
  static const ElemType Zero;
  static const ElemType One;
 };

+#if ROCM_VERSION >= 40300
+// Up until ROCm 4.2 miopenReduceTensor() required alpha/beta to be the same data
+// type as the input type. This differs from cudnnReduceTensor() and other
+// MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16).
+template <>
+struct ReduceConsts<half> {
+  static const float Zero;
+  static const float One;
+};
+#endif
+
 inline double ClampMiopenBatchNormEpsilon(double epsilon) {
  if (epsilon < MIOPEN_BN_MIN_EPSILON) {
    if (MIOPEN_BN_MIN_EPSILON - epsilon > FLT_EPSILON)
--- a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc
+++ b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc
@ -186,14 +186,9 @@ Status ReduceKernel<allow_multi_axes>::ReduceKernelShared(
  else
    ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, ReduceTensorIndices));

-  // As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data
-  // type as the input type. This differs from cudnnReduceTensor() and other
-  // MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16).
-  //
-  // NOTE: this workaround can be removed in ROCm 4.3:
-  //       https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914
-  const auto one = Consts<float>::One;
-  const auto zero = Consts<float>::Zero;
+  const auto one = ReduceConsts<HipT>::One;
+  const auto zero = ReduceConsts<HipT>::Zero;
+
  MiopenTensor input_tensor;
  MiopenTensor output_tensor;
  ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X));
@ -515,14 +510,9 @@ Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, Pr
    ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, ReduceTensorIndices));
  }

-  // As of ROCm 4.2, miopenReduceTensor() requires alpha/beta to be the same data
-  // type as the input type. This differs from cudnnReduceTensor() and other
-  // MIOpen/cuDNN APIs where alpha/beta are float when input type is half (float16).
-  //
-  // NOTE: this workaround can be removed in ROCm 4.3:
-  //       https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914
-  const float one = Consts<float>::One; 
-  const float zero = Consts<float>::Zero; 
+  const auto one = ReduceConsts<HipT>::One;
+  const auto zero = ReduceConsts<HipT>::Zero;
+
  MiopenTensor input_tensor;
  MiopenTensor output_tensor;
  ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X));
--- a/tools/ci_build/amd_hipify.py
+++ b/tools/ci_build/amd_hipify.py
@ -286,6 +286,10 @@ def hipify(src_file_path, dst_file_path):

        # CUFFT -> HIPFFT
        s = s.replace('CUFFT', 'HIPFFT')
+
+        # Undo where above hipify steps went too far.
+        s = s.replace('ROCM_VERSION', 'CUDA_VERSION')  # semantically different meanings, cannot hipify
+
    with open(dst_file_path, 'w') as f:
        f.write(s)

--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@ -296,20 +296,7 @@ stages:
      steps:
      - template: get-docker-image-steps.yml
        parameters:
-          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
-          Context: tools/ci_build/github/linux/docker
-          DockerBuildArgs: >-
-            --build-arg TORCH_VERSION=1.8.1
-            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
-            --build-arg BUILD_UID=$(id -u)
-            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
-            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
-            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
-            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
-          Repository: onnxruntimetrainingrocmbuild-torch1.8.1
-      - template: get-docker-image-steps.yml
-        parameters:
-          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
+          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2
          Context: tools/ci_build/github/linux/docker
          DockerBuildArgs: >-
            --build-arg TORCH_VERSION=1.9.0
@ -319,7 +306,20 @@ stages:
            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
-          Repository: onnxruntimetrainingrocmbuild-torch1.9.0
+          Repository: onnxruntimetrainingrocmbuild-torch1.9.0-rocm4.2
+      - template: get-docker-image-steps.yml
+        parameters:
+          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
+          Context: tools/ci_build/github/linux/docker
+          DockerBuildArgs: >-
+            --build-arg TORCH_VERSION=1.9.0
+            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
+            --build-arg BUILD_UID=$(id -u)
+            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
+            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
+            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
+            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+          Repository: onnxruntimetrainingrocmbuild-torch1.9.0-rocm4.3.1

    - job: ROCM_training_wheels
      timeoutInMinutes: 180
@ -331,30 +331,38 @@ stages:
      - ROCm_build_environment
      strategy:
        matrix:
-          Python36 Torch181:
-            PythonVersion: '3.6'
-            TorchVersion: '1.8.1'
-          Python37 Torch181:
-            PythonVersion: '3.7'
-            TorchVersion: '1.8.1'
-          Python38 Torch181:
-            PythonVersion: '3.8'
-            TorchVersion: '1.8.1'
-          Python39 Torch181:
-            PythonVersion: '3.9'
-            TorchVersion: '1.8.1'
-          Python36 Torch190:
+          Python36 Torch190 Rocm42:
            PythonVersion: '3.6'
            TorchVersion: '1.9.0'
-          Python37 Torch190:
+            RocmVersion: '4.2'
+          Python37 Torch190 Rocm42:
            PythonVersion: '3.7'
            TorchVersion: '1.9.0'
-          Python38 Torch190:
+            RocmVersion: '4.2'
+          Python38 Torch190 Rocm42:
            PythonVersion: '3.8'
            TorchVersion: '1.9.0'
-          Python39 Torch190:
+            RocmVersion: '4.2'
+          Python39 Torch190 Rocm42:
            PythonVersion: '3.9'
            TorchVersion: '1.9.0'
+            RocmVersion: '4.2'
+          Python36 Torch190 Rocm431:
+            PythonVersion: '3.6'
+            TorchVersion: '1.9.0'
+            RocmVersion: '4.3.1'
+          Python37 Torch190 Rocm431:
+            PythonVersion: '3.7'
+            TorchVersion: '1.9.0'
+            RocmVersion: '4.3.1'
+          Python38 Torch190 Rocm431:
+            PythonVersion: '3.8'
+            TorchVersion: '1.9.0'
+            RocmVersion: '4.3.1'
+          Python39 Torch190 Rocm431:
+            PythonVersion: '3.9'
+            TorchVersion: '1.9.0'
+            RocmVersion: '4.3.1'
      steps:

      - checkout: self
@ -380,11 +388,11 @@ stages:
              -e NIGHTLY_BUILD \
              -e BUILD_BUILDNUMBER \
              --user onnxruntimedev \
-              onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
+              onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
                /onnxruntime_src/tools/ci_build/build.py \
                  --config Release \
                  --use_rocm \
-                    --rocm_version=4.2 \
+                    --rocm_version=$(RocmVersion) \
                    --rocm_home=/opt/rocm \
                    --nccl_home=/opt/rocm \
                  --update \
@ -435,7 +443,7 @@ stages:
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
               /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh
        displayName: 'Run onnxruntime unit tests (in container)'

@ -458,7 +466,7 @@ stages:
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
              orttraining/tools/ci_test/run_batch_size_test.py \
                --binary_dir /build/Release \
                --model_root training_e2e_test_data/models \
@ -485,7 +493,7 @@ stages:
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
              orttraining/tools/ci_test/run_bert_perf_test.py \
                --binary_dir /build/Release \
                --model_root training_e2e_test_data/models \
@ -513,7 +521,7 @@ stages:
            -e NIGHTLY_BUILD \
            -e BUILD_BUILDNUMBER \
            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
              orttraining/tools/ci_test/run_convergence_test.py \
                --binary_dir /build/Release \
                --model_root training_e2e_test_data/models \
@ -552,7 +560,7 @@ stages:
              -e NIGHTLY_BUILD \
              -e BUILD_BUILDNUMBER \
              -e PythonManylinuxDir=$(PythonManylinuxdir) \
-              onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
+              onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
                /onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh
          workingDirectory: $(Build.SourcesDirectory)

--- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2
--- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
@ -0,0 +1,203 @@
+ARG BASEIMAGE=rocm/pytorch:rocm4.3.1_centos7_py3.6_pytorch_staging
+ARG POLICY=manylinux2014
+ARG PLATFORM=x86_64
+ARG DEVTOOLSET_ROOTPATH=
+ARG LD_LIBRARY_PATH_ARG=
+ARG PREPEND_PATH=
+
+#Build manylinux2014 docker image begin
+FROM $BASEIMAGE AS runtime_base
+ARG POLICY
+ARG PLATFORM
+ARG DEVTOOLSET_ROOTPATH
+ARG LD_LIBRARY_PATH_ARG
+ARG PREPEND_PATH
+LABEL maintainer="The ManyLinux project"
+
+RUN yum remove -y devtoolset\* git\* && conda remove -y cmake
+
+# remove protobuf 2.6.1 from rocm/pytorch:rocm4.2_centos7_py3.6_pytorch
+# it's too old to compile onnx 1.10
+RUN rm -fr /usr/local/bin/protoc \
+           /usr/local/libproto* \
+           /usr/local/include/google \
+           /usr/local/lib/pkgconfig/protobuf*
+
+ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
+ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
+ENV PATH=${PREPEND_PATH}${PATH}
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+
+# first copy the fixup mirrors script, keep the script around
+COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
+
+# setup entrypoint, this will wrap commands with `linux32` with i686 images
+COPY build_scripts/install-entrypoint.sh build_scripts/update-system-packages.sh /build_scripts/
+RUN bash /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
+COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
+ENTRYPOINT ["manylinux-entrypoint"]
+
+COPY build_scripts/install-runtime-packages.sh build_scripts/update-system-packages.sh /build_scripts/
+RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
+
+COPY build_scripts/build_utils.sh /build_scripts/
+
+COPY build_scripts/install-autoconf.sh /build_scripts/
+RUN export AUTOCONF_ROOT=autoconf-2.71 && \
+    export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \
+    export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \
+    manylinux-entrypoint /build_scripts/install-autoconf.sh
+
+COPY build_scripts/install-automake.sh /build_scripts/
+RUN export AUTOMAKE_ROOT=automake-1.16.4 && \
+    export AUTOMAKE_HASH=8a0f0be7aaae2efa3a68482af28e5872d8830b9813a6a932a2571eac63ca1794 && \
+    export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \
+    manylinux-entrypoint /build_scripts/install-automake.sh
+
+COPY build_scripts/install-libtool.sh /build_scripts/
+RUN export LIBTOOL_ROOT=libtool-2.4.6 && \
+    export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \
+    export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
+    manylinux-entrypoint /build_scripts/install-libtool.sh
+
+COPY build_scripts/install-patchelf.sh /build_scripts/
+RUN export PATCHELF_VERSION=0.13 && \
+    export PATCHELF_HASH=60c6aeadb673de9cc1838b630c81f61e31c501de324ef7f1e8094a2431197d09 && \
+    export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \
+    manylinux-entrypoint /build_scripts/install-patchelf.sh
+
+COPY build_scripts/install-libxcrypt.sh /build_scripts/
+RUN export LIBXCRYPT_VERSION=4.4.23 && \
+    export LIBXCRYPT_HASH=9cbbcb795ed5d121a1613eb0e40c77173b53e15a746796fc7cd7bd71cfd64533 && \
+    export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
+    export PERL_ROOT=perl-5.34.0 && \
+    export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
+    export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
+    manylinux-entrypoint /build_scripts/install-libxcrypt.sh
+
+COPY build_scripts/install-protobuf.sh /build_scripts/
+RUN export PROTOBUF_VERSION=3.17.3 && \
+    export PROTOBUF_ROOT=protobuf-all-${PROTOBUF_VERSION} && \
+    export PROTOBUF_HASH=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 && \
+    export PROTOBUF_DOWNLOAD_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION} && \
+    manylinux-entrypoint /build_scripts/install-protobuf.sh
+
+FROM runtime_base AS build_base
+COPY build_scripts/install-build-packages.sh /build_scripts/
+RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
+
+
+FROM build_base AS build_git
+COPY build_scripts/build-git.sh /build_scripts/
+RUN export GIT_ROOT=git-2.32.0 && \
+    export GIT_HASH=6038f06d396ba9dab2eee541c7db6e7f9f847f181ec62f3d8441893f8c469398 && \
+    export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
+    manylinux-entrypoint /build_scripts/build-git.sh
+
+
+FROM build_base AS build_swig
+COPY build_scripts/build-swig.sh /build_scripts/
+RUN export SWIG_ROOT=swig-4.0.2 && \
+    export SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc && \
+    export SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/${SWIG_ROOT} && \
+    export PCRE_ROOT=pcre-8.44 && \
+    export PCRE_HASH=aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728 && \
+    export PCRE_DOWNLOAD_URL=https://ftp.pcre.org/pub/pcre && \
+    manylinux-entrypoint /build_scripts/build-swig.sh
+
+
+FROM build_base AS build_cpython
+COPY build_scripts/build-sqlite3.sh /build_scripts/
+RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3360000 && \
+    export SQLITE_AUTOCONF_HASH=bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b69bbce3 && \
+    export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \
+    manylinux-entrypoint /build_scripts/build-sqlite3.sh
+
+COPY build_scripts/build-openssl.sh /build_scripts/
+RUN export OPENSSL_ROOT=openssl-1.1.1k && \
+    export OPENSSL_HASH=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 && \
+    export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
+    manylinux-entrypoint /build_scripts/build-openssl.sh
+
+COPY build_scripts/build-cpython.sh /build_scripts/
+
+
+FROM build_cpython AS build_cpython36
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.14
+
+
+FROM build_cpython AS build_cpython37
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.11
+
+
+FROM build_cpython AS build_cpython38
+COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.11
+
+
+FROM build_cpython AS build_cpython39
+COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.6
+
+FROM build_cpython AS all_python
+COPY --from=build_cpython36 /opt/_internal /opt/_internal/
+COPY --from=build_cpython37 /opt/_internal /opt/_internal/
+COPY --from=build_cpython38 /opt/_internal /opt/_internal/
+COPY --from=build_cpython39 /opt/_internal /opt/_internal/
+RUN hardlink -cv /opt/_internal
+
+
+FROM runtime_base
+COPY --from=build_git /manylinux-rootfs /
+COPY --from=build_swig /manylinux-rootfs /
+COPY --from=build_cpython /manylinux-rootfs /
+COPY --from=all_python /opt/_internal /opt/_internal/
+COPY build_scripts/finalize.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/python-tag-abi-tag.py \
+     build_scripts/requirements3.6.txt \
+     build_scripts/requirements3.7.txt \
+     build_scripts/requirements3.8.txt \
+     build_scripts/requirements3.9.txt \
+     build_scripts/requirements-base-tools.txt \
+     /build_scripts/
+COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
+RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+
+ENV SSL_CERT_FILE=/opt/_internal/certs.pem
+
+CMD ["/bin/bash"]
+
+#Build manylinux2014 docker image end
+
+ARG PYTHON_VERSION=3.6
+ARG TORCH_VERSION=1.9.0
+ARG INSTALL_DEPS_EXTRA_ARGS
+
+#Add our own dependencies
+ADD scripts /tmp/scripts
+RUN cd /tmp/scripts && \
+    /tmp/scripts/manylinux/install_centos.sh && \
+    /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.6 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.7 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.8 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
+     rm -rf /tmp/scripts
+
+# remove protobuf to prevent ambiguity which is used for onnxruntime build
+RUN rm -fr /usr/local/bin/protoc \
+           /usr/local/libproto* \
+           /usr/local/include/google \
+           /usr/local/lib/pkgconfig/protobuf*
+
+ARG BUILD_UID=1001
+ARG BUILD_USER=onnxruntimedev
+RUN adduser --uid $BUILD_UID $BUILD_USER
+WORKDIR /home/$BUILD_USER
+USER $BUILD_USER
+ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH