diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 028a94f7b9..c87caf2138 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -272,9 +272,10 @@ def download_test_data(build_dir, src_url, expected_md5, azure_sas_key): return True def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure_sas_key): - """Sets up the test data, downloading it if needed.""" - if not download_test_data(build_dir, test_data_url, test_data_checksum, azure_sas_key): - raise BuildError("Failed to set up test data.") + if test_data_url is not None: + """Sets up the test data, downloading it if needed.""" + if not download_test_data(build_dir, test_data_url, test_data_checksum, azure_sas_key): + raise BuildError("Failed to set up test data.") # create a shortcut for test models if there is a 'models' folder in build_dir if is_windows(): @@ -706,8 +707,9 @@ def main(): update_submodules(source_dir) if args.enable_onnx_tests or args.download_test_data: - if not args.test_data_url or not args.test_data_checksum: - raise UsageError("The test_data_url and test_data_checksum arguments are required.") + if args.download_test_data: + if not args.test_data_url or not args.test_data_checksum: + raise UsageError("The test_data_url and test_data_checksum arguments are required.") setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key) path_to_protoc_exe = None diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index d3d79cf674..ab99fbfb88 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -4,7 +4,23 @@ jobs: steps: - template: templates/set-test-data-variables-step.yml - - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_tvm --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"' + - task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_tvm"' displayName: 'Command Line Script' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml index aaa2f055b1..d4f1d5636a 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml @@ -4,7 +4,23 @@ jobs: steps: - template: templates/set-test-data-variables-step.yml - - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -x "--test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"' + - task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory)' displayName: 'Command Line Script' - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 014bd3d2f1..02a21ff505 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -4,7 +4,23 @@ jobs: steps: - template: templates/set-test-data-variables-step.yml - - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x "--test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"' + - task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)' displayName: 'Command Line Script' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py new file mode 100755 index 0000000000..8bc45abb1a --- /dev/null +++ b/tools/ci_build/github/download_test_data.py @@ -0,0 +1,43 @@ +#!/usr/bin/python3 + +import urllib.request +import json +import subprocess +import os +import argparse +from urllib.parse import urlparse + +def get_azure_region(): + req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01') + req.add_header('Metadata', 'true') + body = urllib.request.urlopen(req).read() + body = json.loads(body.decode('utf-8')) + return body['compute']['location'] + +def parse_arguments(): + parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.") + parser.add_argument("--test_data_url", help="Test data URL.") + return parser.parse_args() + + +def get_server_hostname(): + #should be northcentralus or centralus + azure_location=get_azure_region() + print(azure_location) + if azure_location == 'centralus': + hostname='onnxruntimetestdata' + elif azure_location == 'northcentralus': + hostname='onnxruntimetestdata2' + else: + print('warning: no local data cache for azure region %s' % azure_location) + hostname='onnxruntimetestdata2' + return hostname + +args = parse_arguments() +hostname=get_server_hostname() +url=args.test_data_url.replace('onnxruntimetestdata', hostname) +print('data url=%s' % url) +subprocess.run(['./azcopy','cp', '--log-level','ERROR', url,'.'],check=True) +os.makedirs('models',exist_ok=True) +local_file_name = os.path.basename(urlparse(url).path) +subprocess.run(['unzip', '-qd','models','20190327.zip']) \ No newline at end of file