Remove git and python packages from the docker images used by Zip-Nuget-Java-Nodejs Packaging Pipeline (#11651)

This commit is contained in:
Changming Sun 2022-06-03 20:00:54 -07:00 committed by GitHub
parent 3c1dd9514d
commit d5e34acb82
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 246 additions and 145 deletions

View file

@ -675,7 +675,9 @@ def get_config_build_dir(build_dir, config):
return os.path.join(build_dir, config)
def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, shell=False, env={}, python_path=None):
def run_subprocess(
args, cwd=None, capture_stdout=False, dll_path=None, shell=False, env={}, python_path=None, cuda_home=None
):
if isinstance(args, str):
raise ValueError("args should be a sequence of strings, not a string")
@ -691,6 +693,10 @@ def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, shell=Fa
my_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path
else:
my_env["LD_LIBRARY_PATH"] = dll_path
# Add nvcc's folder to PATH env so that our cmake file can find nvcc
if cuda_home:
my_env["PATH"] = os.path.join(cuda_home, "bin") + os.pathsep + my_env["PATH"]
if python_path:
if "PYTHONPATH" in my_env:
my_env["PYTHONPATH"] += os.pathsep + python_path
@ -907,7 +913,8 @@ def generate_build_tree(
add_default_definition(cmake_extra_defines, "onnxruntime_DEV_MODE", use_dev_mode(args))
if args.use_cuda:
add_default_definition(cmake_extra_defines, "onnxruntime_USE_CUDA", "ON")
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version)
if args.cuda_version:
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version)
# TODO: this variable is not really needed
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_HOME", cuda_home)
add_default_definition(cmake_extra_defines, "onnxruntime_CUDNN_HOME", cudnn_home)
@ -1233,6 +1240,7 @@ def generate_build_tree(
"-DCMAKE_BUILD_TYPE={}".format(config),
],
cwd=config_build_dir,
cuda_home=cuda_home,
)

0
tools/ci_build/github/apple/build_host_protoc.sh Normal file → Executable file
View file

View file

@ -49,19 +49,14 @@ jobs:
- template: templates/set-version-number-variables-step.yml
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.4.0-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecuda11build
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80'
workingDirectory: $(Build.SourcesDirectory)
Dockerfile: tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/x64/default/cpu
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=nvidia/cuda:11.4.3-cudnn8-devel-centos7"
Repository: onnxruntimecuda11centosbuild
- script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/build_cuda_c_api_package.sh
workingDirectory: $(Build.SourcesDirectory)
displayName: 'Build and Test'
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
parameters:

View file

@ -44,80 +44,23 @@ parameters:
default: 'default'
jobs:
- job: Linux_C_API_Packaging_CPU_x64
workspace:
clean: all
timeoutInMinutes: 210
strategy:
matrix:
ARCH_x86_64:
BASE_IMAGE: centos:7
ONNXRUNTIME_ARCH: x64
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
NODEJS_BINDING_ARCH: x64
ARCH_AARCH64:
BASE_IMAGE: arm64v8/centos:7
ONNXRUNTIME_ARCH: aarch64
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
NODEJS_BINDING_ARCH: arm64
pool: 'Linux-CPU'
steps:
- template: set-version-number-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=$(BASE_IMAGE)"
Repository: onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH)
- template: c-api-linux-cpu.yml
parameters:
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }}
BaseImage: 'centos:7'
OnnxruntimeArch: 'x64'
OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
OnnxruntimeNodejsBindingArch: 'x64'
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm -e CFLAGS="$(ONNXRUNTIME_CFLAGS)" -e CXXFLAGS="$(ONNXRUNTIME_CXXFLAGS)" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) /bin/bash -c "python3 \
/onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-$(ONNXRUNTIME_ARCH)"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x
mkdir $(Build.ArtifactStagingDirectory)/testdata
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
ls -al $(Build.ArtifactStagingDirectory)
displayName: 'Create Artifacts'
- template: java-api-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: 'linux-$(ONNXRUNTIME_ARCH)'
buildConfig: 'Release'
artifactName: 'onnxruntime-java-linux-$(ONNXRUNTIME_ARCH)'
version: '$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.so'
nativeLibraryName: 'libonnxruntime4j_jni.so'
- template: c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)-$(OnnxRuntimeVersion)'
artifactNameNoVersionString: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: '$(NODEJS_BINDING_ARCH)'
os: 'linux'
artifactName: 'drop-onnxruntime-nodejs-linux-$(ONNXRUNTIME_ARCH)'
- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- template: c-api-linux-cpu.yml
parameters:
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }}
BaseImage: 'arm64v8/centos:7'
OnnxruntimeArch: 'aarch64'
OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
OnnxruntimeNodejsBindingArch: 'arm64'
- job: MacOS_C_API_Packaging_CPU_x64
workspace:
@ -304,6 +247,7 @@ jobs:
pool: 'Win-CPU-2021'
dependsOn:
- Linux_C_API_Packaging_CPU_x64
- Linux_C_API_Packaging_CPU_aarch64
- MacOS_C_API_Packaging_CPU_x64
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
@ -387,6 +331,7 @@ jobs:
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
dependsOn:
- Linux_C_API_Packaging_CPU_x64
- Linux_C_API_Packaging_CPU_aarch64
- MacOS_C_API_Packaging_CPU_x64
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
@ -591,6 +536,7 @@ jobs:
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
dependsOn:
- Linux_C_API_Packaging_CPU_x64
- Linux_C_API_Packaging_CPU_aarch64
- MacOS_C_API_Packaging_CPU_x64
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
- Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }}

View file

@ -0,0 +1,84 @@
# This file contains the ADO job that build libonnxruntime.so on Linux
parameters:
- name: AdditionalBuildFlags
displayName: Additional build flags for build.py
type: string
default: ''
- name: BaseImage
type: string
- name: OnnxruntimeArch
type: string
- name: OnnxruntimeCFlags
type: string
- name: OnnxruntimeCXXFlags
type: string
- name: OnnxruntimeNodejsBindingArch
type: string
jobs:
- job: Linux_C_API_Packaging_CPU_${{parameters.OnnxruntimeArch}}
workspace:
clean: all
timeoutInMinutes: 210
pool: 'Linux-CPU'
steps:
- template: set-version-number-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}"
Repository: onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}}
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm -e CFLAGS="${{parameters.OnnxruntimeCFlags}}" -e CXXFLAGS="${{parameters.OnnxruntimeCXXFlags}}" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}} /bin/bash -c "python3 \
/onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-${{parameters.OnnxruntimeArch}}"
workingDirectory: $(Build.SourcesDirectory)
displayName: 'Build'
- script: |
set -e -x
mkdir $(Build.ArtifactStagingDirectory)/testdata
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
ls -al $(Build.ArtifactStagingDirectory)
displayName: 'Create Artifacts'
- template: java-api-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: 'linux-${{parameters.OnnxruntimeArch}}'
buildConfig: 'Release'
artifactName: 'onnxruntime-java-linux-${{parameters.OnnxruntimeArch}}'
version: '$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.so'
nativeLibraryName: 'libonnxruntime4j_jni.so'
- template: c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-linux-${{parameters.OnnxruntimeArch}}-$(OnnxRuntimeVersion)'
artifactNameNoVersionString: 'onnxruntime-linux-${{parameters.OnnxruntimeArch}}'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: '${{parameters.OnnxruntimeNodejsBindingArch}}'
os: 'linux'
artifactName: 'drop-onnxruntime-nodejs-linux-${{parameters.OnnxruntimeArch}}'
- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

View file

@ -0,0 +1,10 @@
#!/bin/bash
export CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all"
export CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all"
docker run --gpus all -e CFLAGS -e CXXFLAGS -e NVIDIA_VISIBLE_DEVICES=all --rm --volume \
$BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \
--volume /data/models:/build/models:ro --volume /data/onnx:/data/onnx:ro -e NIGHTLY_BUILD onnxruntimecuda11centosbuild \
python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION \
--cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION \
--cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80'

View file

@ -0,0 +1,11 @@
#!/bin/bash
set -e -x
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
echo "installing for CentOS version : $os_major_version"
yum install -y centos-release-scl-rh
yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-pip
pip3 install --upgrade pip
localedef -i en_US -f UTF-8 en_US.UTF-8

View file

@ -34,27 +34,13 @@ function GetFile {
return $?
}
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
SYS_LONG_BIT=$(getconf LONG_BIT)
mkdir -p /tmp/src
GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)
DISTRIBUTOR=$(lsb_release -i -s)
if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then
LIBDIR="lib64"
else
LIBDIR="lib"
fi
cd /tmp/src
echo "Installing cmake"
GetFile https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-`uname -m`.tar.gz /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz
tar -zxf /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz --strip=1 -C /usr
GetFile https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-`uname -m`.tar.gz /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz
tar -zxf /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz --strip=1 -C /usr
echo "Installing Ninja"
GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz
@ -81,20 +67,5 @@ GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/sr
unzip /tmp/src/gradle-6.3-bin.zip
mv /tmp/src/gradle-6.3 /usr/local/gradle
if ! [ -x "$(command -v protoc)" ]; then
GetFile https://github.com/protocolbuffers/protobuf/archive/v3.18.1.tar.gz /tmp/src/v3.18.1.tar.gz
tar -xf /tmp/src/v3.18.1.tar.gz -C /tmp/src
cd /tmp/src/protobuf-3.18.1
cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo -DCMAKE_INSTALL_LIBDIR=lib64
make -j$(getconf _NPROCESSORS_ONLN)
make install
fi
export ONNX_ML=1
export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
pip3 install -r ${0/%install_deps\.sh/requirements\.txt}
cd /
rm -rf /tmp/src

View file

@ -0,0 +1,20 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline
ARG BASEIMAGE=centos:7
FROM $BASEIMAGE
ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV LANG=en_US.utf8
ENV LC_ALL=en_US.utf8
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev
RUN adduser --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

View file

@ -0,0 +1,11 @@
#!/bin/bash
set -e -x
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
echo "installing for CentOS version : $os_major_version"
yum install -y centos-release-scl-rh
yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-11-binutils devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-gcc-gfortran python3 python3-pip
pip3 install --upgrade pip
localedef -i en_US -f UTF-8 en_US.UTF-8

View file

@ -0,0 +1,71 @@
#!/bin/bash
set -e -x
# Download a file from internet
function GetFile {
local uri=$1
local path=$2
local force=${3:-false}
local download_retries=${4:-5}
local retry_wait_time_seconds=${5:-30}
if [[ -f $path ]]; then
if [[ $force = false ]]; then
echo "File '$path' already exists. Skipping download"
return 0
else
rm -rf $path
fi
fi
if [[ -f $uri ]]; then
echo "'$uri' is a file path, copying file to '$path'"
cp $uri $path
return $?
fi
echo "Downloading $uri"
# Use aria2c if available, otherwise use curl
if command -v aria2c > /dev/null; then
aria2c -q -d $(dirname $path) -o $(basename $path) "$uri"
else
curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail
fi
return $?
}
mkdir -p /tmp/src
cd /tmp/src
echo "Installing cmake"
GetFile https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-`uname -m`.tar.gz /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz
tar -zxf /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz --strip=1 -C /usr
echo "Installing Ninja"
GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz
tar -zxf ninja-linux.tar.gz
cd ninja-1.10.0
cmake -Bbuild-cmake -H.
cmake --build build-cmake
mv ./build-cmake/ninja /usr/bin
echo "Installing Node.js"
CPU_ARCH=`uname -m`
if [[ "$CPU_ARCH" = "x86_64" ]]; then
NODEJS_ARCH=x64
elif [[ "$CPU_ARCH" = "aarch64" ]]; then
NODEJS_ARCH=arm64
else
NODEJS_ARCH=$CPU_ARCH
fi
GetFile https://nodejs.org/dist/v16.14.2/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz /tmp/src/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz
tar --strip 1 -xf /tmp/src/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz -C /usr
cd /tmp/src
GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip
unzip /tmp/src/gradle-6.3-bin.zip
mv /tmp/src/gradle-6.3 /usr/local/gradle
cd /
rm -rf /tmp/src

View file

@ -1,22 +0,0 @@
#!/bin/bash
set -e -x
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
CPU_ARCH=`uname -m`
PACKAGES_TO_INSTALL="which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-devel python3-pip"
if ! rpm -q --quiet epel-release ; then
if [[ "$CPU_ARCH" = "x86_64" ]]; then
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm https://repo.ius.io/ius-release-el$os_major_version.rpm
PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git224"
else
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm
PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git"
fi
fi
echo "installing for os major version : $os_major_version"
yum install -y centos-release-scl-rh
yum install -y $PACKAGES_TO_INSTALL
pip3 install --upgrade pip
localedef -i en_US -f UTF-8 en_US.UTF-8

View file

@ -1,4 +0,0 @@
numpy
git+http://github.com/onnx/onnx.git@be76ca7148396176784ba8733133b9fb1186ea0d#egg=onnx
protobuf
flatbuffers

View file

0
tools/ci_build/github/pai/pai_get_thread.sh Normal file → Executable file
View file