saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-14 20:48:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Disable downloading test data on Linux (#3581)

Browse source
This commit is contained in:
Changming Sun 2020-04-18 15:54:58 -07:00 committed by GitHub
parent 3e884b4b6b
commit d68245853e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 22 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
setup.py
View file

@ -102,7 +102,7 @@ try:
logger.info('copying %s -> %s', source, dest)
copyfile(source, dest)
result = subprocess.run(['patchelf', '--print-needed', dest], check=True, stdout=subprocess.PIPE, universal_newlines=True)
cuda_dependencies = ['libcublas.so', 'libcudnn.so', 'libcudart.so', 'libcurand.so']
cuda_dependencies = ['libcublas.so', 'libcudnn.so', 'libcudart.so', 'libcurand.so', 'libcufft.so']
to_preload = []
args = ['patchelf', '--debug']
for line in result.stdout.split('\n'):

38
tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
View file

@ -9,6 +9,7 @@
jobs:
- job: Manylinux2010_py_Wheels
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-CPU
@ -40,22 +41,7 @@ jobs:
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: CmdLine@2
displayName: 'Download azcopy'
inputs:
script: |
curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
tar -zxvf azcopy.tar.gz --strip 1
workingDirectory: $(Build.BinariesDirectory)
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)
- task: CmdLine@2
inputs:
script: |
@ -64,7 +50,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
@ -84,6 +70,7 @@ jobs:
- template: templates/clean-agent-build-directory-step.yml
- job: Manylinux2010_py_GPU_Wheels
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-GPU-CUDA10
@ -116,21 +103,6 @@ jobs:
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: CmdLine@2
displayName: 'Download azcopy'
inputs:
script: |
curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
tar -zxvf azcopy.tar.gz --strip 1
workingDirectory: $(Build.BinariesDirectory)
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)
- task: CmdLine@2
inputs:
script: |
@ -139,7 +111,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2

4
tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
View file

@ -15,7 +15,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
@ -39,7 +39,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:

16
tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml
View file

@ -1,23 +1,9 @@
jobs:
- job: Linux_CI_GPU_TENSORRT_Dev
pool: Linux-GPU-CUDA10
timeoutInMinutes: 90
steps:
- template: templates/set-test-data-variables-step.yml
- task: CmdLine@2
displayName: 'Download azcopy'
inputs:
script: |
curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
tar -zxvf azcopy.tar.gz --strip 1
workingDirectory: $(Build.BinariesDirectory)
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)
# Latest TensorRT container only supports ubuntu18.04
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d tensorrt -r $(Build.BinariesDirectory) -p 3.6 -x "--build_wheel"'

16
tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml
View file

@ -15,22 +15,6 @@ jobs:
continueOnError: true
condition: always()
- task: CmdLine@2
displayName: 'Download azcopy'
inputs:
script: |
curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
tar -zxvf azcopy.tar.gz --strip 1
workingDirectory: $(Build.BinariesDirectory)
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d ngraph -r $(Build.BinariesDirectory) -x "--use_ngraph --build_wheel"'
displayName: 'Command Line Script'

2
tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml
View file

@ -39,7 +39,7 @@ jobs:
- task: CmdLine@2
displayName: 'Run docker image'
inputs:
script: docker run --rm --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh
script: docker run --rm --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- template: templates/component-governance-component-detection-steps.yml

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml
View file

@ -62,7 +62,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml
View file

@ -37,7 +37,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml
View file

@ -79,7 +79,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
View file

@ -102,7 +102,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

6
tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
View file

@ -74,7 +74,7 @@ jobs:
- job: 'Linux_CI_GPU_Dev'
workspace:
clean: all
pool: $(AgentPoolLinux)
pool: 'Linux-GPU-CUDA10'
steps:
- template: ../../templates/set-version-number-variables-step.yml
- template: ../../templates/linux-set-variables-and-download.yml
@ -86,7 +86,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
- script: |
set -e -x
mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64
@ -236,5 +236,5 @@ jobs:
- template: test_linux.yml
parameters:
AgentPool : $(AgentPoolLinux)
AgentPool : 'Linux-GPU-CUDA10'
TestGPU : 'true'

4
tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
View file

@ -29,7 +29,9 @@ jobs:
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy scipy
mkdir $(Build.SourcesDirectory)\$(BuildConfig)
mkdir $(Build.BinariesDirectory)\$(BuildConfig)
mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models C:\local\models
mklink /D /J $(Build.BinariesDirectory)\models C:\local\models
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'

16
tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml
View file

@ -12,19 +12,3 @@ steps:
continueOnError: true
condition: always()
- task: CmdLine@2
displayName: 'Download azcopy'
inputs:
script: |
curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
tar -zxvf azcopy.tar.gz --strip 1
workingDirectory: $(Build.BinariesDirectory)
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

2
tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml
View file

@ -66,7 +66,7 @@ jobs:
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{ parameters.BuildArch }}-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'

1
tools/ci_build/github/linux/run_dockerbuild.sh
View file

@ -105,6 +105,7 @@ fi
DOCKER_RUN_PARAMETER="--name onnxruntime-$BUILD_DEVICE \
--volume $SOURCE_ROOT:/onnxruntime_src \
--volume $BUILD_DIR:/build \
--volume /data/models:/build/models:ro \
--volume $HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx"
if [ $BUILD_DEVICE = "openvino" ] && [[ $BUILD_EXTR_PAR == *"--use_openvino GPU_FP"* ]]; then