Avoid downloading test data into C:\ (#1562)

This commit is contained in:
Changming Sun 2019-08-05 19:53:15 -07:00 committed by GitHub
parent 05bbb3065c
commit 7ee8aca1bf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 105 additions and 74 deletions

View file

@ -24,7 +24,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)
@ -86,7 +86,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -156,7 +156,7 @@ jobs:
displayName: 'Build and Test OnnxRuntime'
inputs:
script: |
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
workingDirectory: '$(Build.BinariesDirectory)'
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml

View file

@ -16,7 +16,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -16,7 +16,7 @@ jobs:
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
# There are some tests in 20190130.zip that TensorRT can't run. Instead here use 20181210 opset8 for TensorRT test.
arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip
arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -27,7 +27,7 @@ jobs:
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
# nGraph provider fails on the latest 20190729.zip test. revert back to previous zip file until failures can be investigated
arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20190419.zip
arguments: --test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20190419.zip --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -31,7 +31,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrlNoContribOps)
arguments: --test_data_url $(TestDataUrlNoContribOps) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -26,7 +26,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -26,7 +26,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -26,7 +26,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -32,7 +32,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrlNoContribOps)
arguments: --test_data_url $(TestDataUrlNoContribOps) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -19,7 +19,7 @@ jobs:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus
arguments: --test_data_url $(TestDataUrlNoContribOps) --azure_region centralus --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/local/bin/python3'
workingDirectory: $(Build.BinariesDirectory)
- script: |

View file

@ -12,7 +12,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'

View file

@ -12,7 +12,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'
@ -26,7 +26,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev_x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'
@ -41,7 +41,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_Arm64_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64 --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'

View file

@ -12,7 +12,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'
@ -26,7 +26,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev_x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --x86'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'

View file

@ -13,7 +13,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: ${{ parameters.DoCompliance }}
@ -27,7 +27,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_Dev_x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'

View file

@ -13,7 +13,7 @@ jobs:
parameters:
AgentPool : $(AgentPoolWin)
JobName: 'Windows_CI_GPU_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --msvc_toolset=14.11'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'

View file

@ -25,7 +25,7 @@ jobs:
- script: |
@echo "Running build.py --update"
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data
displayName: 'Download Test Data'

View file

@ -24,7 +24,7 @@ steps:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl)
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -13,6 +13,6 @@ steps:
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --azure_region centralus
arguments: --test_data_url $(TestDataUrl) --azure_region centralus --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '/usr/local/bin/python3'
workingDirectory: $(Build.BinariesDirectory)

View file

@ -11,7 +11,7 @@ steps:
displayName: 'Download test data and generate cmake config'
inputs:
script: |
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update ${{parameters.buildAdditionalParams}}
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --update ${{parameters.buildAdditionalParams}}
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1
@ -29,7 +29,7 @@ steps:
displayName: 'Test ${{parameters.buildConfig}}'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test ${{parameters.buildAdditionalParams}}'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{parameters.buildConfig}} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --build_shared_lib --enable_onnx_tests --test ${{parameters.buildAdditionalParams}}'
workingFolder: '$(Build.BinariesDirectory)'
- task: PublishTestResults@2

View file

@ -26,19 +26,25 @@ steps:
# downloadDirectory: '$(Build.BinariesDirectory)\python'
# Temporary bypass of artifacts permission issue
- task: PowerShell@2
- task: CmdLine@1
displayName: 'Download azcopy'
inputs:
filename: 'AzCopy.exe'
arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe /Dest:$(Build.BinariesDirectory)\azcopy.exe'
- task: CmdLine@1
displayName: 'Download python'
inputs:
targetType: 'inline'
script: 'Invoke-WebRequest -OutFile installer.exe https://onnxruntimeinstaller.blob.core.windows.net/conda-installer/installer.exe'
workingDirectory: '$(Build.BinariesDirectory)'
filename: 'AzCopy.exe'
arguments: '/Y /Source:https://onnxruntimetestdata.blob.core.windows.net/models/Miniconda3-4.7.10-Windows-x86_64.exe /Dest:$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe'
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\installer.exe'
filename: '$(Build.BinariesDirectory)\Miniconda3-4.7.10-Windows-x86_64.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
timeoutInMinutes: 10
- task: BatchScript@1
displayName: 'setup env'
inputs:
@ -52,24 +58,13 @@ steps:
arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy'
timeoutInMinutes: 10
- task: CmdLine@1
displayName: 'Download cmake'
- task: PythonScript@0
displayName: 'Download test data'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
- task: PowerShell@2
displayName: 'Download OpenCppCoverage installer'
continueOnError: true
inputs:
targetType: 'inline'
script: '
New-Item -Path "$(Build.BinariesDirectory)\installer" -ItemType "directory"
New-Item -Path "$(Build.BinariesDirectory)\installer\opencppcoverage" -ItemType "directory"
Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe https://onnxruntimeinstaller.blob.core.windows.net/opencppcovergae-installer/OpenCppCoverageSetup-x64-0.9.7.0.exe
'
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe'
workingDirectory: $(Build.BinariesDirectory)
- task: CmdLine@1
continueOnError: true

View file

@ -4,7 +4,7 @@ jobs:
AgentPool : 'Win-CPU'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --gen_doc'
JobName: 'Windows_CI_Dev'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -4,7 +4,7 @@ jobs:
AgentPool : 'Win-GPU-CUDA10'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --msvc_toolset=14.11'
JobName: 'Windows_CI_GPU_Dev'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -28,7 +28,7 @@ jobs:
displayName: 'Download test data and generate cmake config'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --update --msvc_toolset=14.11'
workingDirectory: "$(Build.BinariesDirectory)"
- task: VSBuild@1
@ -45,7 +45,7 @@ jobs:
displayName: 'Test Debug'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test'
workingFolder: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build C# Debug'
@ -81,7 +81,7 @@ jobs:
displayName: 'Test Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test'
workingFolder: "$(Build.BinariesDirectory)"
- task: VSBuild@1

View file

@ -14,7 +14,7 @@ jobs:
displayName: 'Download test data and generate cmake config'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --build_shared_lib --build_csharp --enable_onnx_tests --update'
workingDirectory: "$(Build.BinariesDirectory)"
- task: VSBuild@1
@ -31,7 +31,7 @@ jobs:
displayName: 'Test Debug'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test'
workingFolder: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build C# Debug'
@ -66,7 +66,7 @@ jobs:
displayName: 'Test Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test'
workingFolder: "$(Build.BinariesDirectory)"
- task: VSBuild@1

View file

@ -17,7 +17,7 @@ jobs:
displayName: 'Download test data and generate cmake config'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --gen_doc --update'
workingDirectory: "$(Build.BinariesDirectory)"
- task: VSBuild@1
displayName: 'Build Debug'
@ -33,7 +33,7 @@ jobs:
displayName: 'Test Debug'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_ngraph --use_full_protobuf --build_shared_lib --gen_doc --test'
workingFolder: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build Release'
@ -49,7 +49,7 @@ jobs:
displayName: 'Test Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_ngraph --use_full_protobuf --build_shared_lib --enable_onnx_tests --test'
workingFolder: "$(Build.BinariesDirectory)"
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0

View file

@ -8,7 +8,7 @@ jobs:
AgentPool : 'Win-CPU'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum)'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --build_csharp --enable_onnx_tests'
JobName: 'Windows_CI_Dev'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -4,7 +4,7 @@ jobs:
AgentPool : 'Win-CPU'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86'
JobName: 'Windows_CI_Dev_x86'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -9,7 +9,7 @@ jobs:
AgentPool : 'Win-CPU'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrlNoContribOps) --test_data_checksum $(TestDataChecksum) --disable_contrib_ops --enable_msvc_static_runtime --x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --disable_contrib_ops --enable_msvc_static_runtime --x86'
JobName: 'Windows_CI_Dev_x86'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -1,12 +1,16 @@
#!/usr/bin/python3
import urllib.request
import json
import subprocess
import os
import sys
import shutil
import argparse
from urllib.parse import urlparse
def is_windows():
return sys.platform.startswith("win")
def get_azure_region():
req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01')
req.add_header('Metadata', 'true')
@ -18,28 +22,60 @@ def parse_arguments():
parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.")
parser.add_argument("--test_data_url", help="Test data URL.")
parser.add_argument("--azure_region", help="Azure region")
parser.add_argument("--build_dir", required=True, help="Path to the build directory.")
return parser.parse_args()
def get_server_hostname(azure_location):
if azure_location is None:
#should be northcentralus or centralus
azure_location=get_azure_region()
azure_location = get_azure_region()
print("This VM is in azure location: %s" % azure_location)
if azure_location == 'centralus':
hostname='onnxruntimetestdata'
hostname = 'onnxruntimetestdata'
elif azure_location == 'northcentralus':
hostname='onnxruntimetestdata2'
hostname = 'onnxruntimetestdata2'
else:
print('warning: no local data cache for azure region %s' % azure_location)
hostname='onnxruntimetestdata2'
hostname = 'onnxruntimetestdata2'
return hostname
def download_and_unzip(build_dir, url, dest_folder):
print("Downloading %s" % url)
dest_folder = os.path.join(build_dir, dest_folder)
subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True)
os.makedirs(dest_folder,exist_ok=True)
local_file_name = os.path.join(build_dir, os.path.basename(urlparse(url).path))
if is_windows():
print("unzip %s" % local_file_name)
if shutil.which('7z'): # 7-Zip
subprocess.run(['7z','x', local_file_name, '-y', '-o' + dest_folder], check=True)
elif shutil.which('7za'): # 7-Zip standalone
subprocess.run(['7za', 'x', local_file_name, '-y', '-o' + dest_folder], check=True)
else:
log.error("No unzip tool for use")
sys.exit(1)
else:
subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True)
os.unlink(local_file_name)
args = parse_arguments()
hostname=get_server_hostname(args.azure_region)
url=args.test_data_url.replace('onnxruntimetestdata', hostname)
hostname = get_server_hostname(args.azure_region)
url = args.test_data_url.replace('onnxruntimetestdata', hostname)
print('data url=%s' % url)
subprocess.run(['./azcopy','cp', '--log-level','ERROR', url,'.'],check=True)
os.makedirs('models',exist_ok=True)
local_file_name = os.path.basename(urlparse(url).path)
subprocess.run(['unzip', '-qd','models',local_file_name])
download_and_unzip(args.build_dir, url, 'models')
if is_windows():
url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip'
url = url.replace('onnxruntimetestdata', hostname)
download_and_unzip(args.build_dir, url, 'cmake_temp')
dest_dir = os.path.join(args.build_dir,'cmake')
if os.path.exists(dest_dir):
print('deleting %s' % dest_dir)
shutil.rmtree(dest_dir)
shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe'
url = url.replace('onnxruntimetestdata', hostname)
dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage')
os.makedirs(dest_folder,exist_ok=True)
subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True)