Remove openmp related packaging pipeline (#6991)

1. Remove openmp related packaging pipelines and build jobs.
2. Set continueOnError to true for the TSAUpload tasks. Their service is unstable recently.
3. Update Ubuntu 16 docker images to Ubuntu 18, in prepare for getting C++17 support
4. Cherry-pick the changes in 1.7.1 to the master: updating CFLAGS/CXXFLAGS to strip out debug symbols
This commit is contained in:
Changming Sun 2021-03-12 10:02:59 -08:00 committed by GitHub
parent c588d5d13a
commit 4161758058
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 58 additions and 423 deletions

View file

@ -30,10 +30,6 @@ set (CMAKE_C_STANDARD 99)
include(CheckCXXCompilerFlag)
include(CheckLanguage)
# CentOS compiler is old but it does allow certain C++14 features
# such as lambda captures and they are convenient
# On the other hand it does not allow some others.
# So we cant' regulate simply with the standard.
set(CMAKE_CXX_STANDARD 14)
set_property(GLOBAL PROPERTY USE_FOLDERS ON)

View file

@ -53,7 +53,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc

View file

@ -1,34 +0,0 @@
parameters:
- name: RunOnnxRuntimeTests
displayName: Run Tests?
type: boolean
default: true
- name: DoCompliance
displayName: Run Compliance Tasks?
type: boolean
default: true
- name: DoEsrp
displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
type: boolean
default: false
- name: IsReleaseBuild
displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release.
type: boolean
default: false
variables:
PackageName: 'Microsoft.ML.OnnxRuntime.OpenMP'
jobs:
- template: templates/c-api-cpu.yml
parameters:
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.OpenMP'
AdditionalBuildFlags: '--use_openmp'
AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos'

View file

@ -11,10 +11,10 @@ jobs:
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- task: CmdLine@2
inputs:
@ -29,8 +29,8 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
python3 /onnxruntime_src/tools/ci_build/build.py \
onnxruntimecpubuild \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
--skip_submodule_sync \
@ -39,7 +39,8 @@ jobs:
--build_wheel \
--enable_onnx_tests \
--enable_symbolic_shape_infer_tests \
--enable_pybind --build_java --build_nodejs
--build_java --build_nodejs \
--cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: PublishTestResults@2

View file

@ -38,10 +38,10 @@ jobs:
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- task: CmdLine@2
displayName: Build full onnxruntime and generate ORT format test files
@ -54,7 +54,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
onnxruntimecpubuild \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh
workingDirectory: $(Build.SourcesDirectory)
@ -70,8 +70,8 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
python3 /onnxruntime_src/tools/ci_build/build.py \
onnxruntimecpubuild \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug\
--skip_submodule_sync \
@ -93,7 +93,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
onnxruntimecpubuild \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
--build-directory /build/without_type_reduction \
--reduced-ops-config /home/onnxruntimedev/.test_data/required_ops.ort_models.config \
@ -115,7 +115,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
onnxruntimecpubuild \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
--build-directory /build/with_type_reduction \
--reduced-ops-config /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config \
@ -141,7 +141,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
onnxruntimecpubuild \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
--build-directory /build/with_type_reduction_globally_allowed_types \
--reduced-ops-config /home/onnxruntimedev/.test_data/globally_allowed_types.config \
@ -170,7 +170,7 @@ jobs:
-e BUILD_REASON=$(Build.Reason) \
-e BUILD_BRANCH=$(Build.SourceBranch) \
-e DASHBOARD_MYSQL_ORT_PASSWORD=$(dashboard-mysql-ort-password) \
onnxruntimecentoscpubuild \
onnxruntimecpubuild \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh
workingDirectory: $(Build.SourcesDirectory)

View file

@ -20,7 +20,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" --rm \
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \

View file

@ -19,7 +19,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" --rm \
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \

View file

@ -24,7 +24,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" --rm \
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \

View file

@ -11,10 +11,10 @@ jobs:
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- task: CmdLine@2
inputs:
@ -29,8 +29,8 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecentoscpubuild \
python3 /onnxruntime_src/tools/ci_build/build.py \
onnxruntimecpubuild \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
--skip_submodule_sync \
@ -38,7 +38,8 @@ jobs:
--parallel \
--build_wheel \
--enable_onnx_tests \
--disable_contrib_ops --disable_ml_ops
--disable_contrib_ops --disable_ml_ops \
--cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: PublishTestResults@2

View file

@ -23,7 +23,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: '-o ubuntu16.04 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"'
RunDockerBuildArgs: '-o ubuntu18.04 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"'
- template: templates/component-governance-component-detection-steps.yml
parameters :

View file

@ -16,13 +16,13 @@ jobs:
- task: CmdLine@2
displayName: 'Build docker image'
inputs:
script: docker build --pull -t onnxruntime-server-ubuntu16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server .
script: docker build --pull -t onnxruntime-server-ubuntu18.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=18.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
displayName: 'Run docker image'
inputs:
script: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh
script: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu18.04 /bin/bash /onnxruntime_src/ci/run.sh
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- template: templates/component-governance-component-detection-steps.yml

View file

@ -39,15 +39,15 @@ jobs:
- template: ../../templates/set-version-number-variables-step.yml
- template: ../../templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx && docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \
/onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_nodejs \
--skip_submodule_sync --parallel --build_shared_lib && cd /onnxruntime_src/nodejs && npm pack"
workingDirectory: $(Build.SourcesDirectory)

View file

@ -53,7 +53,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc

View file

@ -88,7 +88,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
/bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=11.0 --cuda_home=/usr/local/cuda-11.0 --cudnn_home=/usr/local/cuda-11.0 --enable_onnx_tests --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc && cd /build/Release && make install DESTDIR=/build/linux-x64"
- script: |
set -e -x

View file

@ -36,10 +36,10 @@ jobs:
- ${{ if eq(parameters['TestGPU'], 'false') }}:
- template: ../../templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- script: |
set -e -x
@ -48,7 +48,7 @@ jobs:
$(Build.BinariesDirectory) \
nuget-artifact \
$(NuGetPackageVersionNumber) \
onnxruntimecentoscpubuild \
onnxruntimecpubuild \
true
displayName: 'Run Package Test (x64) on CentOS7'
env:

View file

@ -7,7 +7,7 @@ jobs:
JobName: 'Onnxruntime_Linux_GPU_Training'
SubmoduleCheckoutMode: 'recursive'
RunDockerBuildArgs: >
-o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)
-o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory)
-x "
--enable_training
--config $(buildConfig)

View file

@ -14,7 +14,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: |
-o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \
-o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory) \
-t onnxruntime_e2e_test_image \
-x " \
--config RelWithDebInfo \

View file

@ -14,7 +14,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: |
-o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \
-o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory) \
-t onnxruntime_distributed_tests_image \
-x " \
--config RelWithDebInfo \

View file

@ -16,7 +16,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: >
-o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)
-o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory)
-t onnxruntime_perf_test_image
-x "
--config RelWithDebInfo

View file

@ -53,16 +53,16 @@ jobs:
- template: set-version-number-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \
docker run --rm -e CC=/opt/rh/devtoolset-9/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-9/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \
/onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)

View file

@ -53,16 +53,17 @@ jobs:
- template: set-version-number-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecentoscpubuild
Repository: onnxruntimecpubuild
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \
docker run --rm -e CC=/opt/rh/devtoolset-9/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-9/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \
/onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
@ -383,7 +384,7 @@ jobs:
command: restore
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2

View file

@ -54,4 +54,5 @@ steps:
displayName: 'TSA Upload'
inputs:
tsaVersion: TsaV2
codeBaseName: 'onnxruntime_master'
codeBaseName: 'onnxruntime_master'
continueOnError: true

View file

@ -2,7 +2,7 @@ parameters:
AgentPool : 'Linux-CPU-2019'
JobName : 'Linux_CI_Dev'
SubmoduleCheckoutMode: ''
RunDockerBuildArgs: '-o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"'
RunDockerBuildArgs: '-o ubuntu18.04 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"'
DoNodejsPack: 'false'
DoNugetPack: 'false'
NuPackScript: ''

View file

@ -142,81 +142,6 @@ stages:
- template: clean-agent-build-directory-step.yml
- job: Linux_py_Wheels_openmp
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-CPU
strategy:
matrix:
Python36:
PythonVersion: '3.6'
Python37:
PythonVersion: '3.7'
Python38:
PythonVersion: '3.8'
Python39:
PythonVersion: '3.9'
steps:
- checkout: self
clean: true
submodules: recursive
- template: set-python-manylinux-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecpubuild
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
-e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
-e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
onnxruntimecpubuild \
$(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Release \
--skip_submodule_sync \
--parallel \
--enable_lto \
--build_wheel \
--use_openmp \
--enable_onnx_tests \
--wheel_name_suffix=openmp \
${{ parameters.build_py_parameters }} \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(PythonManylinuxIncludeDir) PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- template: clean-agent-build-directory-step.yml
- ${{ if eq(parameters.enable_linux_gpu, true) }}:
- job: Linux_py_GPU_Wheels
@ -496,183 +421,6 @@ stages:
displayName: 'Clean Agent Directories'
condition: always()
- job: Windows_py_Wheels_openmp
pool: 'Win-CPU-2021'
strategy:
matrix:
Python36:
PythonVersion: '3.6'
Python37:
PythonVersion: '3.7'
Python38:
PythonVersion: '3.8'
Python39:
PythonVersion: '3.9'
variables:
MsbuildArguments: '-maxcpucount'
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
EnvSetupScript: setup_env.bat
buildArch: x64
setVcvars: true
BuildConfig: 'Release'
GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
timeoutInMinutes: 120
workspace:
clean: all
steps:
- checkout: self
clean: true
submodules: recursive
- template: telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
versionSpec: $(PythonVersion)
addToPath: true
architecture: 'x64'
- template: set-nightly-build-option-variable-step.yml
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
python -m pip uninstall -y onnx -qq
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- task: PythonScript@0
displayName: 'BUILD'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--enable_lto
--disable_rtti
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "Visual Studio 16 2019"
--enable_pybind
--use_openmp
--enable_onnx_tests
${{ parameters.build_py_parameters }}
--parallel
$(TelemetryOption)
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption) --wheel_name_suffix=openmp'
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- script: |
rename *.whl *.zip
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
displayName: 'Rename file extension for codesign validation'
- task: DeleteFiles@1
displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: |
**/*.obj
**/*.pdb
**/*.dll
- task: PythonScript@0
displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON'
condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--enable_lto
--disable_rtti
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "Visual Studio 16 2019"
--enable_pybind
--use_openmp
--enable_onnx_tests
--update
--cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON
${{ parameters.build_py_parameters }}
--parallel
$(TelemetryOption)
workingDirectory: '$(Build.BinariesDirectory)'
#Manually set msBuildCommandline so that we can also set CAExcludePath
- task: SDLNativeRules@2
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
inputs:
userProvideBuildInfo: msBuildInfo
msBuildVersion: 16.0
msBuildArchitecture: x64
msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="x64" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64'
continueOnError: true
- task: TSAUpload@1
condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.7')), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
displayName: 'TSA Upload'
inputs:
tsaVersion: TsaV2
codeBaseName: 'onnxruntime_master'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- ${{ if eq(parameters.enable_windows_gpu, true) }}:
- job: Windows_py_GPU_Wheels
workspace:
@ -839,6 +587,7 @@ stages:
inputs:
tsaVersion: TsaV2
codeBaseName: 'onnxruntime_master'
continueOnError: true
- template: component-governance-component-detection-steps.yml
parameters:
@ -906,63 +655,6 @@ stages:
parameters:
condition: 'succeeded'
- job: MacOS_py_Wheels_openmp
workspace:
clean: all
pool:
vmImage: 'macOS-10.14'
strategy:
matrix:
Python36:
PythonVersion: '3.6'
Python37:
PythonVersion: '3.7'
Python38:
PythonVersion: '3.8'
Python39:
PythonVersion: '3.9'
steps:
- checkout: self
clean: true
submodules: recursive
- task: UsePythonVersion@0
displayName: 'Use Python'
inputs:
versionSpec: $(PythonVersion)
- script: |
set -e
pushd .
cd $(Build.SourcesDirectory)/cmake/external/protobuf
cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo
make -j$(getconf _NPROCESSORS_ONLN)
make install
popd
export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH
export ONNX_ML=1
export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
brew install libomp
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --config Release --skip_onnx_tests --build_wheel ${{ parameters.build_py_parameters }} --wheel_name_suffix=openmp
displayName: 'Command Line Script'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)/Release/dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- ${{ if eq(parameters.enable_linux_arm, true) }}:
- job: Linux_ARM_py_Wheels

View file

@ -1,11 +0,0 @@
FROM centos:7
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev
RUN adduser --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER
ENV PATH /usr/local/gradle/bin:$PATH

View file

@ -1,13 +0,0 @@
ARG OS_VERSION=27
FROM fedora:${OS_VERSION}
ARG PYTHON_VERSION=3.6
# nvidia-container-runtime
ENV NVIDIA_VISIBLE_DEVICES=all \
NVIDIA_DRIVER_CAPABILITIES=compute,utility \
NVIDIA_REQUIRE_CUDA="cuda>=9.2" \
PATH="/usr/lib64/ccache:${PATH}"
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_fedora_gpu.sh && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts

View file

@ -5,6 +5,7 @@
set -e
set -x
export PATH=/opt/python/cp37-cp37m/bin:$PATH
# Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details
# to find kernels. If the hashes from the registration details are incorrect we will produce a model that will break
@ -21,7 +22,7 @@ python3 /onnxruntime_src/tools/ci_build/build.py \
--parallel \
--build_wheel \
--skip_tests \
--enable_pybind
--enable_pybind --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so
# Install the ORT python wheel
python3 -m pip install --user /build/Debug/dist/*

View file

@ -7,7 +7,7 @@
set -e
set -x
export PATH=/opt/python/cp37-cp37m/bin:$PATH
USAGE_TEXT="Usage:
-b|--build-directory <build directory>
Specifies the build directory. Required.

View file

@ -5,7 +5,7 @@
set -e
set -x
export PATH=/opt/python/cp37-cp37m/bin:$PATH
# Create an empty file to be used with build --include_ops_by_config, which will include no operators at all
echo -n > /home/onnxruntimedev/.test_data/include_no_operators.config