diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5fc6d98993..6dae802164 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -675,7 +675,9 @@ def get_config_build_dir(build_dir, config): return os.path.join(build_dir, config) -def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, shell=False, env={}, python_path=None): +def run_subprocess( + args, cwd=None, capture_stdout=False, dll_path=None, shell=False, env={}, python_path=None, cuda_home=None +): if isinstance(args, str): raise ValueError("args should be a sequence of strings, not a string") @@ -691,6 +693,10 @@ def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, shell=Fa my_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path else: my_env["LD_LIBRARY_PATH"] = dll_path + # Add nvcc's folder to PATH env so that our cmake file can find nvcc + if cuda_home: + my_env["PATH"] = os.path.join(cuda_home, "bin") + os.pathsep + my_env["PATH"] + if python_path: if "PYTHONPATH" in my_env: my_env["PYTHONPATH"] += os.pathsep + python_path @@ -907,7 +913,8 @@ def generate_build_tree( add_default_definition(cmake_extra_defines, "onnxruntime_DEV_MODE", use_dev_mode(args)) if args.use_cuda: add_default_definition(cmake_extra_defines, "onnxruntime_USE_CUDA", "ON") - add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version) + if args.cuda_version: + add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version) # TODO: this variable is not really needed add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_HOME", cuda_home) add_default_definition(cmake_extra_defines, "onnxruntime_CUDNN_HOME", cudnn_home) @@ -1233,6 +1240,7 @@ def generate_build_tree( "-DCMAKE_BUILD_TYPE={}".format(config), ], cwd=config_build_dir, + cuda_home=cuda_home, ) diff --git a/tools/ci_build/github/apple/build_host_protoc.sh b/tools/ci_build/github/apple/build_host_protoc.sh old mode 100644 new mode 100755 diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index f06ac88eaf..39a604e215 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -49,19 +49,14 @@ jobs: - template: templates/set-version-number-variables-step.yml - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.4.0-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecuda11build - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80' - workingDirectory: $(Build.SourcesDirectory) + Dockerfile: tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x64/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=nvidia/cuda:11.4.3-cudnn8-devel-centos7" + Repository: onnxruntimecuda11centosbuild + + - script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/build_cuda_c_api_package.sh + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Build and Test' - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 88babe5dca..43064c0cef 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -44,80 +44,23 @@ parameters: default: 'default' jobs: -- job: Linux_C_API_Packaging_CPU_x64 - workspace: - clean: all - timeoutInMinutes: 210 - strategy: - matrix: - ARCH_x86_64: - BASE_IMAGE: centos:7 - ONNXRUNTIME_ARCH: x64 - ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' - ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' - NODEJS_BINDING_ARCH: x64 - ARCH_AARCH64: - BASE_IMAGE: arm64v8/centos:7 - ONNXRUNTIME_ARCH: aarch64 - ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' - ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' - NODEJS_BINDING_ARCH: arm64 - pool: 'Linux-CPU' - steps: - - template: set-version-number-variables-step.yml - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=$(BASE_IMAGE)" - Repository: onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) +- template: c-api-linux-cpu.yml + parameters: + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + BaseImage: 'centos:7' + OnnxruntimeArch: 'x64' + OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' + OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' + OnnxruntimeNodejsBindingArch: 'x64' - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm -e CFLAGS="$(ONNXRUNTIME_CFLAGS)" -e CXXFLAGS="$(ONNXRUNTIME_CXXFLAGS)" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) /bin/bash -c "python3 \ - /onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-$(ONNXRUNTIME_ARCH)" - workingDirectory: $(Build.SourcesDirectory) - - - script: | - set -e -x - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' - - - template: java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'linux-$(ONNXRUNTIME_ARCH)' - buildConfig: 'Release' - artifactName: 'onnxruntime-java-linux-$(ONNXRUNTIME_ARCH)' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.so' - nativeLibraryName: 'libonnxruntime4j_jni.so' - - - template: c-api-artifacts-package-and-publish-steps-posix.yml - parameters: - buildConfig: 'Release' - artifactName: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)-$(OnnxRuntimeVersion)' - artifactNameNoVersionString: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)' - libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - - - template: nodejs-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: '$(NODEJS_BINDING_ARCH)' - os: 'linux' - artifactName: 'drop-onnxruntime-nodejs-linux-$(ONNXRUNTIME_ARCH)' - - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() +- template: c-api-linux-cpu.yml + parameters: + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + BaseImage: 'arm64v8/centos:7' + OnnxruntimeArch: 'aarch64' + OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' + OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' + OnnxruntimeNodejsBindingArch: 'arm64' - job: MacOS_C_API_Packaging_CPU_x64 workspace: @@ -304,6 +247,7 @@ jobs: pool: 'Win-CPU-2021' dependsOn: - Linux_C_API_Packaging_CPU_x64 + - Linux_C_API_Packaging_CPU_aarch64 - MacOS_C_API_Packaging_CPU_x64 - Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }} @@ -387,6 +331,7 @@ jobs: breakCodesignValidationInjection: ${{ parameters.DoEsrp }} dependsOn: - Linux_C_API_Packaging_CPU_x64 + - Linux_C_API_Packaging_CPU_aarch64 - MacOS_C_API_Packaging_CPU_x64 - Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }} @@ -591,6 +536,7 @@ jobs: breakCodesignValidationInjection: ${{ parameters.DoEsrp }} dependsOn: - Linux_C_API_Packaging_CPU_x64 + - Linux_C_API_Packaging_CPU_aarch64 - MacOS_C_API_Packaging_CPU_x64 - Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }} diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml new file mode 100644 index 0000000000..68c521c8b1 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -0,0 +1,84 @@ +# This file contains the ADO job that build libonnxruntime.so on Linux +parameters: +- name: AdditionalBuildFlags + displayName: Additional build flags for build.py + type: string + default: '' + +- name: BaseImage + type: string + +- name: OnnxruntimeArch + type: string + +- name: OnnxruntimeCFlags + type: string + +- name: OnnxruntimeCXXFlags + type: string + +- name: OnnxruntimeNodejsBindingArch + type: string + +jobs: +- job: Linux_C_API_Packaging_CPU_${{parameters.OnnxruntimeArch}} + workspace: + clean: all + timeoutInMinutes: 210 + pool: 'Linux-CPU' + steps: + - template: set-version-number-variables-step.yml + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}" + Repository: onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}} + + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm -e CFLAGS="${{parameters.OnnxruntimeCFlags}}" -e CXXFLAGS="${{parameters.OnnxruntimeCXXFlags}}" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}} /bin/bash -c "python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \ + --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-${{parameters.OnnxruntimeArch}}" + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Build' + + - script: | + set -e -x + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata + ls -al $(Build.ArtifactStagingDirectory) + displayName: 'Create Artifacts' + + - template: java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'linux-${{parameters.OnnxruntimeArch}}' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-linux-${{parameters.OnnxruntimeArch}}' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' + + - template: c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: 'onnxruntime-linux-${{parameters.OnnxruntimeArch}}-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-linux-${{parameters.OnnxruntimeArch}}' + libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' + + - template: nodejs-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: '${{parameters.OnnxruntimeNodejsBindingArch}}' + os: 'linux' + artifactName: 'drop-onnxruntime-nodejs-linux-${{parameters.OnnxruntimeArch}}' + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() \ No newline at end of file diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh new file mode 100755 index 0000000000..bb4f273912 --- /dev/null +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -0,0 +1,10 @@ +#!/bin/bash +export CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" +export CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" +docker run --gpus all -e CFLAGS -e CXXFLAGS -e NVIDIA_VISIBLE_DEVICES=all --rm --volume \ +$BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ +--volume /data/models:/build/models:ro --volume /data/onnx:/data/onnx:ro -e NIGHTLY_BUILD onnxruntimecuda11centosbuild \ +python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ +--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION \ +--cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION \ +--cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80' diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile similarity index 100% rename from tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile rename to tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh new file mode 100755 index 0000000000..cc736e21f5 --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e -x + +os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) + +echo "installing for CentOS version : $os_major_version" +yum install -y centos-release-scl-rh +yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-pip + +pip3 install --upgrade pip +localedef -i en_US -f UTF-8 en_US.UTF-8 diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_deps.sh similarity index 61% rename from tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh rename to tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_deps.sh index 3e6e5245db..38f41034d3 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_deps.sh @@ -34,27 +34,13 @@ function GetFile { return $? } - - -os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) - -SYS_LONG_BIT=$(getconf LONG_BIT) mkdir -p /tmp/src -GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.) - -DISTRIBUTOR=$(lsb_release -i -s) - -if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then - LIBDIR="lib64" -else - LIBDIR="lib" -fi cd /tmp/src echo "Installing cmake" -GetFile https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-`uname -m`.tar.gz /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz -tar -zxf /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz --strip=1 -C /usr +GetFile https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-`uname -m`.tar.gz /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz +tar -zxf /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz --strip=1 -C /usr echo "Installing Ninja" GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz @@ -81,20 +67,5 @@ GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/sr unzip /tmp/src/gradle-6.3-bin.zip mv /tmp/src/gradle-6.3 /usr/local/gradle -if ! [ -x "$(command -v protoc)" ]; then - GetFile https://github.com/protocolbuffers/protobuf/archive/v3.18.1.tar.gz /tmp/src/v3.18.1.tar.gz - tar -xf /tmp/src/v3.18.1.tar.gz -C /tmp/src - cd /tmp/src/protobuf-3.18.1 - cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo -DCMAKE_INSTALL_LIBDIR=lib64 - make -j$(getconf _NPROCESSORS_ONLN) - make install -fi - -export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - -pip3 install -r ${0/%install_deps\.sh/requirements\.txt} - - cd / rm -rf /tmp/src diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile new file mode 100644 index 0000000000..067444a7df --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline +ARG BASEIMAGE=centos:7 +FROM $BASEIMAGE + +ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV LANG=en_US.utf8 +ENV LC_ALL=en_US.utf8 + +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts + +ARG BUILD_UID=1001 +ARG BUILD_USER=onnxruntimedev +RUN adduser --uid $BUILD_UID $BUILD_USER +WORKDIR /home/$BUILD_USER +USER $BUILD_USER + diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x64/default/cpu/scripts/install_centos.sh new file mode 100755 index 0000000000..2c0925bcba --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x64/default/cpu/scripts/install_centos.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e -x + +os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) + +echo "installing for CentOS version : $os_major_version" +yum install -y centos-release-scl-rh +yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-11-binutils devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-gcc-gfortran python3 python3-pip + +pip3 install --upgrade pip +localedef -i en_US -f UTF-8 en_US.UTF-8 diff --git a/tools/ci_build/github/linux/docker/inference/x64/default/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x64/default/cpu/scripts/install_deps.sh new file mode 100755 index 0000000000..cbbf26c806 --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x64/default/cpu/scripts/install_deps.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -e -x + +# Download a file from internet +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use aria2c if available, otherwise use curl + if command -v aria2c > /dev/null; then + aria2c -q -d $(dirname $path) -o $(basename $path) "$uri" + else + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + fi + + return $? +} +mkdir -p /tmp/src + +cd /tmp/src + +echo "Installing cmake" +GetFile https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-`uname -m`.tar.gz /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz +tar -zxf /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz --strip=1 -C /usr + +echo "Installing Ninja" +GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz +tar -zxf ninja-linux.tar.gz +cd ninja-1.10.0 +cmake -Bbuild-cmake -H. +cmake --build build-cmake +mv ./build-cmake/ninja /usr/bin + +echo "Installing Node.js" +CPU_ARCH=`uname -m` +if [[ "$CPU_ARCH" = "x86_64" ]]; then + NODEJS_ARCH=x64 +elif [[ "$CPU_ARCH" = "aarch64" ]]; then + NODEJS_ARCH=arm64 +else + NODEJS_ARCH=$CPU_ARCH +fi +GetFile https://nodejs.org/dist/v16.14.2/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz /tmp/src/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz +tar --strip 1 -xf /tmp/src/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz -C /usr + +cd /tmp/src +GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip +unzip /tmp/src/gradle-6.3-bin.zip +mv /tmp/src/gradle-6.3 /usr/local/gradle + +cd / +rm -rf /tmp/src \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh deleted file mode 100755 index 3847d86b78..0000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -x - -os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) -CPU_ARCH=`uname -m` -PACKAGES_TO_INSTALL="which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-devel python3-pip" -if ! rpm -q --quiet epel-release ; then - if [[ "$CPU_ARCH" = "x86_64" ]]; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm https://repo.ius.io/ius-release-el$os_major_version.rpm - PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git224" - else - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm - PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git" - fi -fi - -echo "installing for os major version : $os_major_version" -yum install -y centos-release-scl-rh -yum install -y $PACKAGES_TO_INSTALL - -pip3 install --upgrade pip -localedef -i en_US -f UTF-8 en_US.UTF-8 \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt deleted file mode 100644 index aa9f6e926d..0000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -numpy -git+http://github.com/onnx/onnx.git@be76ca7148396176784ba8733133b9fb1186ea0d#egg=onnx -protobuf -flatbuffers diff --git a/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh b/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh old mode 100644 new mode 100755 diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh old mode 100644 new mode 100755 diff --git a/tools/ci_build/github/pai/pai_get_thread.sh b/tools/ci_build/github/pai/pai_get_thread.sh old mode 100644 new mode 100755