mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Delete some unused code in run_dockerbuild.sh and Enable Nuget CUDA tests (#8089)
1. Remove some unused code and simplify tools/ci_build/github/linux/run_dockerbuild.sh. 2. Enable Nuget CUDA tests. The original design was we could leverage Directory.Build.props and let cmake generate the required properties(USE_CUDA/...) there. However, in nuget packaging pipeline we test the package on a different host that doesn't run cmake command and doesn't have the auto-generated Directory.Build.props file.
This commit is contained in:
Changming Sun
GitHub
parent
f6292d9b38
commit
6e2b064aec
20 changed files with 106 additions and 310 deletions
|
|
@ -1,35 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
# build docker image for CPU
|
||||
|
||||
set -x -e
|
||||
|
||||
SOURCE_ROOT=$1
|
||||
BUILD_DIR=$2
|
||||
NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR
|
||||
CurrentOnnxRuntimeVersion=$4
|
||||
DockerImage=$5
|
||||
PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu}
|
||||
RunTestCsharp=${RunTestCsharp:-true}
|
||||
RunTestNative=${RunTestNative:-true}
|
||||
|
||||
OldDir=$(pwd)
|
||||
|
||||
cd $SOURCE_ROOT/tools/ci_build/github/linux/docker
|
||||
|
||||
docker run --gpus all --rm \
|
||||
--volume "$SOURCE_ROOT:/onnxruntime_src" \
|
||||
--volume "$BUILD_DIR:/home/onnxruntimedev" \
|
||||
--volume /data/models:/home/onnxruntimedev/models:ro \
|
||||
-e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \
|
||||
-e "IsReleaseBuild=$ISRELEASEBUILD" \
|
||||
-e "PACKAGENAME=$PACKAGENAME" \
|
||||
-e "RunTestCsharp=$RunTestCsharp" \
|
||||
-e "RunTestNative=$RunTestNative" \
|
||||
-e "BUILD_BINARIESDIRECTORY=/home/onnxruntimedev" \
|
||||
-e "BUILD_SOURCESDIRECTORY=/onnxruntime_src" \
|
||||
"$DockerImage" \
|
||||
/bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
|
||||
/home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
# build docker image for CPU
|
||||
|
||||
set -x -e
|
||||
|
||||
SOURCE_ROOT=$1
|
||||
BUILD_DIR=$2
|
||||
NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR
|
||||
CurrentOnnxRuntimeVersion=$4
|
||||
DockerImage=$5
|
||||
UseCentos7=${6:-false}
|
||||
Arch=${7:-x64} # x32, x64
|
||||
PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime}
|
||||
RunTestCsharp=${RunTestCsharp:-true}
|
||||
RunTestNative=${RunTestNative:-true}
|
||||
PYTHON_VER=3.5
|
||||
IMAGE="ubuntu16.04_$Arch"
|
||||
|
||||
OldDir=$(pwd)
|
||||
|
||||
cd $SOURCE_ROOT/tools/ci_build/github/linux/docker
|
||||
|
||||
|
||||
docker run --rm \
|
||||
--name "onnxruntime-cpu" \
|
||||
--volume "$SOURCE_ROOT:/onnxruntime_src" \
|
||||
--volume "$BUILD_DIR:/home/onnxruntimedev" \
|
||||
--volume /data/models:/home/onnxruntimedev/models:ro \
|
||||
-e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \
|
||||
-e "IsReleaseBuild=$ISRELEASEBUILD" \
|
||||
-e "PACKAGENAME=$PACKAGENAME" \
|
||||
-e "DisableContribOps=$DISABLECONTRIBOPS" \
|
||||
-e "DisableMlOps=$DISABLEMLOPS" \
|
||||
-e "RunTestCsharp=$RunTestCsharp" \
|
||||
-e "RunTestNative=$RunTestNative" \
|
||||
-e "BUILD_BINARIESDIRECTORY=/home/onnxruntimedev" \
|
||||
-e "BUILD_SOURCESDIRECTORY=/onnxruntime_src" \
|
||||
"$DockerImage" \
|
||||
/bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
|
||||
/home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion
|
||||
|
|
@ -3,9 +3,8 @@
|
|||
# Licensed under the MIT License.
|
||||
|
||||
LocalNuGetRepo=$1
|
||||
BuildDir=$3
|
||||
export CurrentOnnxRuntimeVersion=$4
|
||||
IsMacOS=${5:-false}
|
||||
export CurrentOnnxRuntimeVersion=$2
|
||||
IsMacOS=${3:-false}
|
||||
PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime}
|
||||
RunTestCsharp=${RunTestCsharp:-true}
|
||||
RunTestNative=${RunTestNative:-true}
|
||||
|
|
@ -29,7 +28,11 @@ if [ $RunTestCsharp = "true" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
dotnet test $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed
|
||||
if [ $PACKAGENAME = "Microsoft.ML.OnnxRuntime.Gpu" ]; then
|
||||
dotnet test -p:DefineConstants=USE_CUDA $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed
|
||||
else
|
||||
dotnet test $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed
|
||||
fi
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to build or execute the end-to-end test"
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -4,6 +4,6 @@ jobs:
|
|||
AgentPool : 'Linux-Multi-GPU'
|
||||
JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev'
|
||||
# The latest TensorRT container only supports ubuntu20.04 and python 3.8
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d tensorrt -r $(Build.BinariesDirectory) -p 3.8 -x "--enable_multi_device_test"'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d tensorrt -x "--enable_multi_device_test"'
|
||||
DoNugetPack: 'false'
|
||||
ArtifactName: 'drop-linux'
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ jobs:
|
|||
parameters:
|
||||
AgentPool : 'Linux-CPU-2019'
|
||||
JobName: 'Linux_CI_Dev'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -x "--enable_pybind --use_nuphar"'
|
||||
DoNugetPack: 'false'
|
||||
ArtifactName: 'drop-linux'
|
||||
TimeoutInMinutes: 180
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ jobs:
|
|||
parameters:
|
||||
AgentPool : 'Linux-CPU-2019'
|
||||
JobName: 'Linux_CI_Dev'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2021.3 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2021.3 -x "--use_openvino CPU_FP32 --build_wheel"'
|
||||
DoNugetPack: 'false'
|
||||
ArtifactName: 'drop-linux'
|
||||
TimeoutInMinutes: 120
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ jobs:
|
|||
|
||||
- template: templates/run-docker-build-steps.yml
|
||||
parameters:
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2020.2 -x "--use_openvino GPU_FP32 --build_wheel"'
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
|
|
|
|||
|
|
@ -315,7 +315,7 @@ jobs:
|
|||
parameters:
|
||||
AgentPool : 'Onnxruntime-Linux-GPU'
|
||||
TestGPU : 'true'
|
||||
|
||||
|
||||
- job: Publish_NuGet_Package_And_Report
|
||||
workspace:
|
||||
clean: all
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
parameters:
|
||||
AgentPool: 'Linux-CPU'
|
||||
TestGPU: 'false'
|
||||
|
||||
jobs:
|
||||
- job: NuGet_Test_Linux
|
||||
|
|
@ -32,48 +31,25 @@ jobs:
|
|||
parameters:
|
||||
packageFolder: '$(Build.BinariesDirectory)/nuget-artifact'
|
||||
|
||||
- ${{ if eq(parameters['TestGPU'], 'false') }}:
|
||||
- template: ../../templates/get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
||||
Repository: onnxruntimecpubuild
|
||||
|
||||
- script: |
|
||||
set -e -x
|
||||
$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh \
|
||||
$(Build.SourcesDirectory) \
|
||||
$(Build.BinariesDirectory) \
|
||||
nuget-artifact \
|
||||
$(NuGetPackageVersionNumber) \
|
||||
onnxruntimecpubuild \
|
||||
true
|
||||
displayName: 'Run Package Test (x64) on CentOS7'
|
||||
env:
|
||||
OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
|
||||
- ${{ if eq(parameters['TestGPU'], 'true') }}:
|
||||
- template: ../../templates/get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.1-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
|
||||
Repository: onnxruntimecuda11build
|
||||
|
||||
- script: |
|
||||
set -e -x
|
||||
$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh \
|
||||
$(Build.SourcesDirectory) \
|
||||
$(Build.BinariesDirectory) \
|
||||
nuget-artifact \
|
||||
$(NuGetPackageVersionNumber) \
|
||||
onnxruntimecuda11build
|
||||
displayName: 'Run Package Test GPU (x64)'
|
||||
env:
|
||||
OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory)
|
||||
- task: CmdLine@2
|
||||
displayName: 'Create symlink for test models'
|
||||
inputs:
|
||||
script: |
|
||||
ln -sf /data/models $(Build.BinariesDirectory)
|
||||
|
||||
- task: Bash@3
|
||||
displayName: 'Run Package Test'
|
||||
inputs:
|
||||
targetType: filePath
|
||||
filePath: '$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh'
|
||||
arguments: '$(Build.BinariesDirectory)/nuget-artifact $(NuGetPackageVersionNumber)'
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
env:
|
||||
OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory)
|
||||
DisableContribOps: $(DisableContribOps)
|
||||
DisableMlOps: $(DisableMlOps)
|
||||
IsReleaseBuild: $(IsReleaseBuild)
|
||||
|
||||
- template: ../../templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'always'
|
||||
|
|
|
|||
|
|
@ -31,11 +31,8 @@ jobs:
|
|||
packageFolder: '$(Build.BinariesDirectory)/nuget-artifact'
|
||||
|
||||
- script: |
|
||||
brew install libomp
|
||||
$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
|
||||
$(Build.BinariesDirectory)/nuget-artifact \
|
||||
$(Build.SourcesDirectory) \
|
||||
$(Build.BinariesDirectory) \
|
||||
$(NuGetPackageVersionNumber) \
|
||||
${{ parameters.IsMacOS }}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ jobs:
|
|||
JobName: 'Onnxruntime_Linux_GPU_Training'
|
||||
SubmoduleCheckoutMode: 'recursive'
|
||||
RunDockerBuildArgs: >
|
||||
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory)
|
||||
-o ubuntu20.04 -d gpu
|
||||
-t onnxruntime_orttraining_ortmodule_tests_image
|
||||
-e
|
||||
-x "
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ jobs:
|
|||
- template: templates/run-docker-build-steps.yml
|
||||
parameters:
|
||||
RunDockerBuildArgs: |
|
||||
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
|
||||
-o ubuntu20.04 -d gpu \
|
||||
-t onnxruntime_e2e_test_image \
|
||||
-x " \
|
||||
--config RelWithDebInfo \
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ jobs:
|
|||
- template: templates/run-docker-build-steps.yml
|
||||
parameters:
|
||||
RunDockerBuildArgs: |
|
||||
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
|
||||
-o ubuntu20.04 -d gpu \
|
||||
-t onnxruntime_distributed_tests_image \
|
||||
-x " \
|
||||
--config RelWithDebInfo \
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ jobs:
|
|||
- template: templates/run-docker-build-steps.yml
|
||||
parameters:
|
||||
RunDockerBuildArgs: |
|
||||
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
|
||||
-o ubuntu20.04 -d gpu \
|
||||
-t onnxruntime_ortmodule_distributed_tests_image \
|
||||
-x " \
|
||||
--config RelWithDebInfo \
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ jobs:
|
|||
- template: templates/run-docker-build-steps.yml
|
||||
parameters:
|
||||
RunDockerBuildArgs: >
|
||||
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory)
|
||||
-o ubuntu20.04 -d gpu
|
||||
-t onnxruntime_perf_test_image
|
||||
-x "
|
||||
--config RelWithDebInfo
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ parameters:
|
|||
AgentPool : 'Linux-CPU-2019'
|
||||
JobName : 'Linux_CI_Dev'
|
||||
SubmoduleCheckoutMode: ''
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -x "--build_wheel"'
|
||||
DoNodejsPack: 'false'
|
||||
DoNugetPack: 'false'
|
||||
NuPackScript: ''
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
|
||||
|
||||
|
||||
if ! rpm -q --quiet epel-release ; then
|
||||
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm
|
||||
fi
|
||||
|
||||
echo "installing for os major version : $os_major_version"
|
||||
yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel
|
||||
|
||||
# install dotnet core dependencies
|
||||
yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid
|
||||
# install dotnet runtimes
|
||||
yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
|
||||
yum install -y dotnet-sdk-2.2 java-1.8.0-openjdk-devel ccache gcc gcc-c++ python3 python3-devel python3-pip
|
||||
|
||||
# install automatic documentation generation dependencies
|
||||
yum install -y graphviz
|
||||
|
|
@ -3,12 +3,10 @@ set -e
|
|||
while getopts p:d: parameter_Option
|
||||
do case "${parameter_Option}"
|
||||
in
|
||||
p) PYTHON_VER=${OPTARG};;
|
||||
d) DEVICE_TYPE=${OPTARG};;
|
||||
esac
|
||||
done
|
||||
|
||||
PYTHON_VER=${PYTHON_VER:=3.5}
|
||||
# Some Edge devices only have limited disk space, use this option to exclude some package
|
||||
DEVICE_TYPE=${DEVICE_TYPE:=Normal}
|
||||
|
||||
|
|
@ -21,126 +19,48 @@ OS_VERSION=$(lsb_release -r -s)
|
|||
|
||||
SYS_LONG_BIT=$(getconf LONG_BIT)
|
||||
|
||||
PACKAGE_LIST="autotools-dev \
|
||||
automake \
|
||||
build-essential \
|
||||
git apt-transport-https apt-utils \
|
||||
ca-certificates \
|
||||
pkg-config \
|
||||
wget \
|
||||
zlib1g \
|
||||
zlib1g-dev \
|
||||
libssl-dev \
|
||||
curl libcurl4-openssl-dev \
|
||||
autoconf \
|
||||
sudo \
|
||||
gfortran \
|
||||
python3-dev \
|
||||
language-pack-en \
|
||||
liblttng-ust0 \
|
||||
libcurl4 \
|
||||
libssl1.1 \
|
||||
libkrb5-3 \
|
||||
libicu66 \
|
||||
libtinfo-dev \
|
||||
libtool \
|
||||
openssh-server \
|
||||
aria2 \
|
||||
bzip2 \
|
||||
unzip \
|
||||
zip \
|
||||
rsync libunwind8 libpng-dev libexpat1-dev \
|
||||
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
|
||||
openjdk-11-jdk \
|
||||
graphviz"
|
||||
|
||||
#see: https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore21
|
||||
if [ "$OS_VERSION" = "18.04" ]; then
|
||||
PACKAGE_LIST="autotools-dev \
|
||||
automake \
|
||||
build-essential \
|
||||
git apt-transport-https apt-utils \
|
||||
ca-certificates \
|
||||
pkg-config \
|
||||
wget \
|
||||
zlib1g \
|
||||
zlib1g-dev \
|
||||
libssl-dev \
|
||||
curl libcurl4-openssl-dev \
|
||||
autoconf \
|
||||
sudo \
|
||||
gfortran \
|
||||
python3-dev \
|
||||
language-pack-en \
|
||||
liblttng-ust0 \
|
||||
libcurl4 \
|
||||
libssl1.0.0 \
|
||||
libkrb5-3 \
|
||||
libicu60 \
|
||||
libtinfo-dev \
|
||||
libtool \
|
||||
openssh-server \
|
||||
aria2 \
|
||||
bzip2 \
|
||||
unzip \
|
||||
zip \
|
||||
rsync libunwind8 libpng-dev libexpat1-dev \
|
||||
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
|
||||
openjdk-11-jdk \
|
||||
graphviz"
|
||||
else # ubuntu20.04
|
||||
PACKAGE_LIST="autotools-dev \
|
||||
automake \
|
||||
build-essential \
|
||||
git apt-transport-https apt-utils \
|
||||
ca-certificates \
|
||||
pkg-config \
|
||||
wget \
|
||||
zlib1g \
|
||||
zlib1g-dev \
|
||||
libssl-dev \
|
||||
curl libcurl4-openssl-dev \
|
||||
autoconf \
|
||||
sudo \
|
||||
gfortran \
|
||||
python3-dev \
|
||||
language-pack-en \
|
||||
liblttng-ust0 \
|
||||
libcurl4 \
|
||||
libssl1.1 \
|
||||
libkrb5-3 \
|
||||
libicu66 \
|
||||
libtinfo-dev \
|
||||
libtool \
|
||||
openssh-server \
|
||||
aria2 \
|
||||
bzip2 \
|
||||
unzip \
|
||||
zip \
|
||||
rsync libunwind8 libpng-dev libexpat1-dev \
|
||||
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
|
||||
openjdk-11-jdk \
|
||||
graphviz"
|
||||
fi
|
||||
|
||||
if [ $DEVICE_TYPE = "Normal" ]; then
|
||||
PACKAGE_LIST="$PACKAGE_LIST libedit-dev libxml2-dev python3-packaging"
|
||||
fi
|
||||
apt-get update && apt-get install -y --no-install-recommends $PACKAGE_LIST
|
||||
apt-get install -y --no-install-recommends $PACKAGE_LIST
|
||||
|
||||
locale-gen en_US.UTF-8
|
||||
update-locale LANG=en_US.UTF-8
|
||||
|
||||
if [ "$SYS_LONG_BIT" = "64" ]; then
|
||||
mkdir -p /tmp/dotnet
|
||||
aria2c -q -d /tmp/dotnet https://packages.microsoft.com/config/ubuntu/${OS_VERSION}/packages-microsoft-prod.deb
|
||||
dpkg -i /tmp/dotnet/packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y dotnet-sdk-2.1
|
||||
rm -rf /tmp/dotnet
|
||||
fi
|
||||
|
||||
if [ "$OS_VERSION" = "16.04" ]; then
|
||||
exit 1
|
||||
elif [ "$OS_VERSION" = "18.04" ]; then
|
||||
if [ "$PYTHON_VER" != "3.6" ]; then
|
||||
add-apt-repository -y ppa:deadsnakes/ppa
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
python${PYTHON_VER} \
|
||||
python${PYTHON_VER}-dev
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 2
|
||||
update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
|
||||
#TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be
|
||||
#put at /usr/local/. Then there will be two pips.
|
||||
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
|
||||
fi
|
||||
|
||||
else # ubuntu20.04
|
||||
if [ "$PYTHON_VER" != "3.8" ]; then
|
||||
add-apt-repository -y ppa:deadsnakes/ppa
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
python${PYTHON_VER} \
|
||||
python${PYTHON_VER}-dev
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2
|
||||
update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
|
||||
#TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be
|
||||
#put at /usr/local/. Then there will be two pips.
|
||||
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
if [ "$SYS_LONG_BIT" = "64" ]; then
|
||||
|
|
|
|||
|
|
@ -9,15 +9,8 @@ if ! rpm -q --quiet epel-release ; then
|
|||
fi
|
||||
|
||||
echo "installing for os major version : $os_major_version"
|
||||
yum install -y which gdb redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel
|
||||
yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel
|
||||
|
||||
if [ "$os_major_version" == "7" ]; then
|
||||
# install dotnet core dependencies
|
||||
yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid
|
||||
# install dotnet runtimes
|
||||
yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
|
||||
yum install -y dotnet-sdk-2.1
|
||||
fi
|
||||
|
||||
# Install Java
|
||||
# Install automatic documentation generation dependencies
|
||||
|
|
|
|||
|
|
@ -1,29 +1,34 @@
|
|||
#!/bin/bash
|
||||
#This file is only for Linux pipelines that build on ubuntu. All the docker images here are based on ubuntu.
|
||||
#Please don't put CentOS or manylinux2014 related stuffs here.
|
||||
set -e -o -x
|
||||
id
|
||||
SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )"
|
||||
SOURCE_ROOT=$(realpath $SCRIPT_DIR/../../../../)
|
||||
SOURCE_ROOT=$BUILD_SOURCESDIRECTORY
|
||||
SCRIPT_DIR=$BUILD_SOURCESDIRECTORY/tools/ci_build/github/linux
|
||||
BUILD_DIR=$BUILD_BINARIESDIRECTORY
|
||||
|
||||
|
||||
YOCTO_VERSION="4.19"
|
||||
#Training only
|
||||
INSTALL_DEPS_DISTRIBUTED_SETUP=false
|
||||
#Training only
|
||||
ORTMODULE_BUILD=false
|
||||
#Training only
|
||||
USE_CONDA=false
|
||||
ALLOW_RELEASED_ONNX_OPSET_ONLY_ENV="ALLOW_RELEASED_ONNX_OPSET_ONLY="$ALLOW_RELEASED_ONNX_OPSET_ONLY
|
||||
echo "ALLOW_RELEASED_ONNX_OPSET_ONLY environment variable is set as "$ALLOW_RELEASED_ONNX_OPSET_ONLY_ENV
|
||||
|
||||
while getopts c:o:d:r:p:x:a:v:y:t:i:mue parameter_Option
|
||||
while getopts o:d:p:x:v:y:t:i:mue parameter_Option
|
||||
do case "${parameter_Option}"
|
||||
in
|
||||
#android, ubuntu16.04, ubuntu18.04, CentOS7
|
||||
#android, yocto, ubuntu20.04
|
||||
o) BUILD_OS=${OPTARG};;
|
||||
#cpu, gpu, tensorrt
|
||||
#gpu, tensorrt or openvino. It is ignored when BUILD_OS is android or yocto.
|
||||
d) BUILD_DEVICE=${OPTARG};;
|
||||
r) BUILD_DIR=${OPTARG};;
|
||||
#python version: 3.6 3.7 (absence means default 3.6)
|
||||
p) PYTHON_VER=${OPTARG};;
|
||||
# "--build_wheel --use_openblas"
|
||||
x) BUILD_EXTR_PAR=${OPTARG};;
|
||||
# x86 or other, only for ubuntu16.04 os
|
||||
a) BUILD_ARCH=${OPTARG};;
|
||||
# openvino version tag: 2020.3 (OpenVINO EP 2.0 supports version starting 2020.3)
|
||||
v) OPENVINO_VERSION=${OPTARG};;
|
||||
# YOCTO 4.19 + ACL 19.05, YOCTO 4.14 + ACL 19.02
|
||||
|
|
@ -94,30 +99,32 @@ elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then
|
|||
DOCKER_FILE=Dockerfile.ubuntu_tensorrt
|
||||
fi
|
||||
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
|
||||
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64" \
|
||||
--dockerfile $DOCKER_FILE --context .
|
||||
elif [ $BUILD_DEVICE = "openvino" ]; then
|
||||
IMAGE="$BUILD_OS-openvino"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_openvino
|
||||
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
|
||||
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION}" \
|
||||
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \
|
||||
--dockerfile $DOCKER_FILE --context .
|
||||
else
|
||||
IMAGE="$BUILD_OS"
|
||||
IMAGE_OS_VERSION=""
|
||||
if [ $BUILD_OS = "ubuntu18.04" ]; then
|
||||
IMAGE_OS_VERSION="18.04"
|
||||
PYTHON_VER="3.6"
|
||||
elif [ $BUILD_OS = "ubuntu20.04" ]; then
|
||||
IMAGE_OS_VERSION="20.04"
|
||||
PYTHON_VER="3.8"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IMAGE_OS_VERSION=""
|
||||
if [ $BUILD_OS = "ubuntu18.04" ]; then
|
||||
IMAGE_OS_VERSION="18.04"
|
||||
PYTHON_VER="3.6"
|
||||
elif [ $BUILD_OS = "ubuntu20.04" ]; then
|
||||
IMAGE_OS_VERSION="20.04"
|
||||
PYTHON_VER="3.8"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
BUILD_ARGS="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OS_VERSION=${IMAGE_OS_VERSION}"
|
||||
|
||||
if [ $BUILD_DEVICE = "openvino" ]; then
|
||||
IMAGE="$BUILD_OS-openvino"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_openvino
|
||||
BUILD_ARGS+=" --build-arg OPENVINO_VERSION=${OPENVINO_VERSION}"
|
||||
else
|
||||
IMAGE="$BUILD_OS"
|
||||
DOCKER_FILE=Dockerfile.ubuntu
|
||||
fi
|
||||
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
|
||||
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OS_VERSION=${IMAGE_OS_VERSION}" \
|
||||
--dockerfile Dockerfile.ubuntu --context .
|
||||
--docker-build-args="${BUILD_ARGS}" \
|
||||
--dockerfile $DOCKER_FILE --context .
|
||||
fi
|
||||
|
||||
if [ -v EXTRA_IMAGE_TAG ]; then
|
||||
|
|
@ -125,7 +132,6 @@ if [ -v EXTRA_IMAGE_TAG ]; then
|
|||
fi
|
||||
|
||||
set +e
|
||||
mkdir -p ~/.cache/onnxruntime
|
||||
mkdir -p ~/.onnx
|
||||
|
||||
if [ -z "$NIGHTLY_BUILD" ]; then
|
||||
|
|
|
|||
Loading…
Reference in a new issue