Disable test data local cache in Linux CI pipelines

tools/ci_build/build.py

@@ -272,9 +272,10 @@ def download_test_data(build_dir, src_url, expected_md5, azure_sas_key):
     return True
 
 def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure_sas_key):
-    """Sets up the test data, downloading it if needed."""
-    if not download_test_data(build_dir, test_data_url, test_data_checksum, azure_sas_key):
-        raise BuildError("Failed to set up test data.")
+    if test_data_url is not None:
+        """Sets up the test data, downloading it if needed."""
+        if not download_test_data(build_dir, test_data_url, test_data_checksum, azure_sas_key):
+            raise BuildError("Failed to set up test data.")
 
     # create a shortcut for test models if there is a 'models' folder in build_dir
     if is_windows():
@@ -706,8 +707,9 @@ def main():
             update_submodules(source_dir)
 
         if args.enable_onnx_tests or args.download_test_data:
-            if not args.test_data_url or not args.test_data_checksum:
-               raise UsageError("The test_data_url and test_data_checksum arguments are required.")
+            if args.download_test_data:
+                if not args.test_data_url or not args.test_data_checksum:
+                   raise UsageError("The test_data_url and test_data_checksum arguments are required.")
             setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key)
 
         path_to_protoc_exe = None

tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml

@@ -4,7 +4,23 @@ jobs:
   steps:
     - template: templates/set-test-data-variables-step.yml
 
-    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_tvm --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"'
+    - task: CmdLine@2
+      displayName: 'Download azcopy'
+      inputs:
+        script: |
+          curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
+          tar -zxvf azcopy.tar.gz --strip 1
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - task: PythonScript@0
+      displayName: 'Download test data'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+        arguments: --test_data_url $(TestDataUrl)
+        pythonInterpreter: '/usr/bin/python3'
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_tvm"'
       displayName: 'Command Line Script'
 
     - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0

tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml

@@ -4,7 +4,23 @@ jobs:
   steps:
     - template: templates/set-test-data-variables-step.yml
 
-    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -x "--test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"'
+    - task: CmdLine@2
+      displayName: 'Download azcopy'
+      inputs:
+        script: |
+          curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
+          tar -zxvf azcopy.tar.gz --strip 1
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - task: PythonScript@0
+      displayName: 'Download test data'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+        arguments: --test_data_url $(TestDataUrl)
+        pythonInterpreter: '/usr/bin/python3'
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory)'
       displayName: 'Command Line Script'
 
     - template: templates/clean-agent-build-directory-step.yml

tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml

@@ -4,7 +4,23 @@ jobs:
   steps:
     - template: templates/set-test-data-variables-step.yml
 
-    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x "--test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"'
+    - task: CmdLine@2
+      displayName: 'Download azcopy'
+      inputs:
+        script: |
+          curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
+          tar -zxvf azcopy.tar.gz --strip 1
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - task: PythonScript@0
+      displayName: 'Download test data'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+        arguments: --test_data_url $(TestDataUrl)
+        pythonInterpreter: '/usr/bin/python3'
+        workingDirectory: $(Build.BinariesDirectory)
+
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)'
       displayName: 'Command Line Script'
 
     - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0

tools/ci_build/github/download_test_data.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+import urllib.request
+import json
+import subprocess
+import os
+import argparse
+from urllib.parse import urlparse
+
+def get_azure_region():
+    req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01')
+    req.add_header('Metadata', 'true')
+    body = urllib.request.urlopen(req).read()
+    body = json.loads(body.decode('utf-8'))
+    return body['compute']['location']
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.")
+    parser.add_argument("--test_data_url", help="Test data URL.")
+    return parser.parse_args()
+
+
+def get_server_hostname():
+    #should be northcentralus or centralus
+    azure_location=get_azure_region()
+    print(azure_location)
+    if azure_location == 'centralus':
+        hostname='onnxruntimetestdata'
+    elif azure_location == 'northcentralus':
+        hostname='onnxruntimetestdata2'
+    else:
+        print('warning: no local data cache for azure region %s' % azure_location)
+        hostname='onnxruntimetestdata2'
+    return hostname
+
+args = parse_arguments()
+hostname=get_server_hostname()
+url=args.test_data_url.replace('onnxruntimetestdata', hostname)
+print('data url=%s' % url)
+subprocess.run(['./azcopy','cp', '--log-level','ERROR', url,'.'],check=True)
+os.makedirs('models',exist_ok=True)
+local_file_name = os.path.basename(urlparse(url).path)
+subprocess.run(['unzip', '-qd','models','20190327.zip'])