add additional test data set for nuget pipeline (#2448)

* add SAS token to download internal test data for nuget pipeline * update azure endpoint * fix keyvault download step * fix variable declaration for secret group * fix indentation * fix yaml syntax for variables * fix setting secrets for script * fix env synctax * Fix macos pipeline * attempt to add secrets to windows download data * fix mac and win data download * fix windows data download * update test data set url and location
2026-07-24 19:43:35 +00:00 · 2019-11-25 13:08:03 -08:00 · 2019-11-25 13:08:03 -08:00 · 4caf5c9c13
commit 4caf5c9c13
parent e0b51e4d95
10 changed files with 202 additions and 51 deletions
--- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
@ -56,7 +56,6 @@ jobs:
  pool: 
    vmImage: 'macOS-10.13'
  steps:
-  
    - template: templates/mac-set-variables-and-download.yml
    - template: templates/set-version-number-variables-step.yml

--- a/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml
@ -0,0 +1,29 @@
+steps:
+- task: CmdLine@2
+  displayName: 'Clean untagged docker images'
+  inputs:
+    script: |
+      docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true
+      docker container prune -f
+      docker image prune -f
+    workingDirectory: $(Build.BinariesDirectory)
+  continueOnError: true
+  condition: always()
+
+- task: CmdLine@2
+  displayName: 'Download azcopy'
+  inputs:
+    script: |
+      curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux'
+      tar -zxvf azcopy.tar.gz --strip 1
+    workingDirectory: $(Build.BinariesDirectory)
+
+- task: PythonScript@0
+  displayName: 'Download test data'
+  inputs:
+    scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+    arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
+    pythonInterpreter: '/usr/bin/python3'
+    workingDirectory: $(Build.BinariesDirectory)
+  env:
+    Test_Data_Download_Key: $(ort-internal-test-data-sas)
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/mac-set-variables-and-download.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/mac-set-variables-and-download.yml
@ -0,0 +1,18 @@
+steps:
+- task: CmdLine@2
+  displayName: 'Download azcopy'
+  inputs:
+    script: |
+      curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-mac'
+      tar -zxvf azcopy.tar.gz --strip 1
+    workingDirectory: $(Build.BinariesDirectory)
+
+- task: PythonScript@0
+  displayName: 'Download test data'
+  inputs:
+    scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py'
+    arguments: --test_data_url $(TestDataUrl) --azure_region centralus --build_dir $(Build.BinariesDirectory)
+    pythonInterpreter: '/usr/local/bin/python3'
+    workingDirectory: $(Build.BinariesDirectory)
+  env:
+    Test_Data_Download_Key: $(ort-internal-test-data-sas)
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/set-test-data-variables-step.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/set-test-data-variables-step.yml
@ -0,0 +1,11 @@
+# sets variables $(TestDataUrl)
+
+parameters:
+  TestDataUrl: https://ortinternaltestdata.blob.core.windows.net/models/20191122.zip
+
+steps:
+- task: CmdLine@1
+  displayName: 'Set TestDataUrl variable'
+  inputs:
+    filename: echo
+    arguments: '##vso[task.setvariable variable=TestDataUrl;]${{parameters.TestDataUrl}}'
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml
@ -11,9 +11,12 @@ jobs:
  - NuGet_Packaging
  condition: succeeded()
  variables:
-    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
+  - group: ORT_TEST_DATA_SAS
+  - name: OnnxRuntimeBuildDirectory
+    value: '$(Build.BinariesDirectory)'
  steps:
-  - template: ../../templates/linux-set-variables-and-download.yml
+  - template: set-test-data-variables-step.yml
+  - template: linux-set-variables-and-download.yml

  - task: DownloadPipelineArtifact@0
    displayName: 'Download Signed NuGet'
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml
@ -10,9 +10,12 @@ jobs:
  - NuGet_Packaging
  condition: succeeded()
  variables:
-    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
+  - group: ORT_TEST_DATA_SAS
+  - name: OnnxRuntimeBuildDirectory
+    value: '$(Build.BinariesDirectory)'
  steps:
-  - template: ../../templates/mac-set-variables-and-download.yml
+  - template: set-test-data-variables-step.yml
+  - template: mac-set-variables-and-download.yml

  - task: DownloadPipelineArtifact@0
    displayName: 'Download Pipeline Artifact - Signed NuGet'
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
@ -10,14 +10,19 @@ jobs:
  - NuGet_Packaging
  condition: succeeded()
  variables:
-    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
+  - group: ORT_TEST_DATA_SAS
+  - name: OnnxRuntimeBuildDirectory
+    value: '$(Build.BinariesDirectory)'
+
  steps:
-  - template: ../../templates/set-test-data-variables-step.yml
+  - template: set-test-data-variables-step.yml
  - template: ../../templates/windows-build-tools-setup-steps.yml
    parameters:
      EnvSetupScript: 'setup_env.bat'
      buildArch: 'x64'
      setVcvars: true
+      DoDataDownload: false
+  - template: win-download-test-data.yml

  - task: BatchScript@1
    displayName: 'Setup VS2017 env vars'
@ -34,12 +39,6 @@ jobs:
  - template: get-nuget-package-version-as-variable.yml
    parameters:
      packageFolder: '$(Build.BinariesDirectory)\nuget-artifact'
-    
-  - script: |
-     @echo "Running download_test_data.py"
-     $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
-
-    displayName: 'Download Test Data'

  - script: |
     @echo "Running Runtest.bat"
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml
@ -0,0 +1,18 @@
+# Assumes AZCopy and Python download is already done
+steps:
+- task: PythonScript@0
+  displayName: 'Download test data'
+  inputs:
+    scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
+    arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
+    pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe'
+    workingDirectory: $(Build.BinariesDirectory)
+  env:
+    Test_Data_Download_Key: $(ort-internal-test-data-sas)
+
+- task: CmdLine@1
+  continueOnError: true
+  displayName: 'Run OpenCPPCoverage installer'
+  inputs:
+    filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe'
+    arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"'
--- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml
@ -2,6 +2,7 @@

 parameters:
  EnvSetupScript: ''
+  DoDataDownload: true
 steps:
    - task: NuGetToolInstaller@0
      displayName: Use Nuget 4.9
@ -53,6 +54,7 @@ steps:

    - task: PythonScript@0
      displayName: 'Download test data'
+      condition: ${{parameters.DoDataDownload}}
      inputs:
        scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
        arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
@ -62,6 +64,7 @@ steps:
    - task: CmdLine@1
      continueOnError: true
      displayName: 'Run OpenCPPCoverage installer'
+      condition: ${{parameters.DoDataDownload}}
      inputs:
        filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe'
        arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"'    
--- a/tools/ci_build/github/download_test_data.py
+++ b/tools/ci_build/github/download_test_data.py
@ -7,16 +7,39 @@ import sys
 import shutil
 import argparse
 from urllib.parse import urlparse
+from urllib.parse import urljoin
+from urllib.parse import urlsplit
+
+# Hardcoded map of storage account to azure region endpoint
+storage_account_to_endpoint_map = {
+    'onnxruntimetestdata.blob.core.windows.net' : {
+        'centralus' : 'onnxruntimetestdata.blob.core.windows.net',
+        'northcentralus' : 'onnxruntimetestdata2.blob.core.windows.net'
+    },
+    'ortinternaltestdata' : {
+        'centralus' : 'ortinternaltestdata.blob.core.windows.net'
+    }
+}

 def is_windows():
    return sys.platform.startswith("win")

 def get_azure_region():
-    req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01')
-    req.add_header('Metadata', 'true')
-    body = urllib.request.urlopen(req).read()
-    body = json.loads(body.decode('utf-8'))
-    return body['compute']['location']
+    # Try to get azure location for this machine
+    if args.azure_region is not None:
+        return args.azure_region
+    else:
+        azure_region = None
+        try:
+            req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01')
+            req.add_header('Metadata', 'true')
+            body = urllib.request.urlopen(req).read()
+            body = json.loads(body.decode('utf-8'))
+            azure_region = body['compute']['location']
+            print("This VM is in azure region: %s" % azure_region)
+        except:
+            print("Failed to get azure region for this machine.")
+    return azure_region

 def parse_arguments():
    parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.")
@ -27,25 +50,45 @@ def parse_arguments():
    return parser.parse_args()


-def get_server_hostname(azure_location):
-    if azure_location is None:
-      #should be northcentralus or centralus
-      azure_location = get_azure_region()
-    print("This VM is in azure location: %s" % azure_location)
-    if azure_location == 'centralus':
-        hostname = 'onnxruntimetestdata'
-    elif azure_location == 'northcentralus':
-        hostname = 'onnxruntimetestdata2'
+def get_endpoint_by_region(current_endpoint, azure_location):
+    # Find storage account closest to the current region
+    # TODO : Ideally we should replicate same storage account to multiple regions
+    # and choose the endpoint accordigly instead of the current way where
+    # we maintain separate storage accounts per region.
+    if current_endpoint in storage_account_to_endpoint_map:
+        endpoint_to_region_map = storage_account_to_endpoint_map[current_endpoint]
+        if azure_location in endpoint_to_region_map:
+            return endpoint_to_region_map[azure_location]
+        else:
+            print('warning: no local data cache for azure region %s' % azure_location)
+            return current_endpoint
    else:
        print('warning: no local data cache for azure region %s' % azure_location)
-        hostname = 'onnxruntimetestdata2'
-    return hostname
+        return current_endpoint
+    return current_endpoint

+def get_region_based_url(url, azure_location):
+    current_endpoint = urlsplit(url).netloc
+    endpoint_by_region = get_endpoint_by_region(current_endpoint, args.azure_region)
+    url = url.replace(current_endpoint, endpoint_by_region)
+    print("url changed to %s" % url)
+    return url

-def download_and_unzip(build_dir, url, dest_folder):
-    print("Downloading %s" % url)
+def download_and_unzip(build_dir, url, dest_folder, use_token = True):
    dest_folder = os.path.join(build_dir, dest_folder)
-    subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True)
+    # attach the SAS token to the url. Note DO NOT print the url with the token in any logs.
+    token = os.environ.get('Test_Data_Download_Key')
+    if use_token:
+        url_with_token = urljoin(url, token)
+    else:
+        url_with_token = url
+
+    # Download data using AZCopy tool
+    try:
+        subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', '--recursive', url_with_token, build_dir],check=True)
+    except:
+        raise Exception("Downloading data failed. Source: " + url + " Destination: " + build_dir)
+
    os.makedirs(dest_folder,exist_ok=True)
    local_file_name = os.path.join(build_dir, os.path.basename(urlparse(url).path))
    if is_windows():
@ -61,30 +104,55 @@ def download_and_unzip(build_dir, url, dest_folder):
       subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True)
    os.unlink(local_file_name)

+def download_additional_data(build_dir, azure_region):
+    additional_data_url = 'https://onnxruntimetestdata.blob.core.windows.net/models/'
+    url = get_region_based_url(args.test_data_url, azure_region)
+
+    cmake_url = urljoin(additional_data_url, 'cmake-3.15.1-win64-x64.zip')
+    print("Starting download for cmake : " + cmake_url)
+    download_and_unzip(build_dir, cmake_url, 'cmake_temp', False)
+    dest_dir = os.path.join(build_dir,'cmake')
+    if os.path.exists(dest_dir):
+        print('deleting %s' % dest_dir)
+        shutil.rmtree(dest_dir)
+    shutil.move(os.path.join(build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
+
+    # Download OpenCPPCoverageSetup.exe
+    opencpp_url = urljoin(additional_data_url, 'OpenCppCoverageSetup-x64-0.9.7.0.exe')
+    print("Starting download for opencppcoverage " + opencpp_url)
+    dest_folder = os.path.join(build_dir, 'installer','opencppcoverage')
+    os.makedirs(dest_folder,exist_ok=True)
+    subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', opencpp_url, os.path.join(dest_folder,'installer.exe')],check=True)
+
 args = parse_arguments()
 models_folder = 'models'
+
 if args.edge_device:
    dest_folder = os.path.join(args.build_dir, models_folder)
    #For edge device, the model zip file is persist at /mnt/ubuntu/tmp/model.zip
    local_file_name = '/mnt/ubuntu/tmp/model.zip'
    if os.path.exists(local_file_name):
-       subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True)
+        subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True)
+    else:
+        raise Exception(local_file_name + " does not exist on edge device. Downloading test data step failed.")
 else:
-    hostname = get_server_hostname(args.azure_region)
-    url = args.test_data_url.replace('onnxruntimetestdata', hostname)
-    print('data url=%s' % url)
-    download_and_unzip(args.build_dir, url, models_folder)
-    if is_windows():
-        url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip'
-        url = url.replace('onnxruntimetestdata', hostname)
-        download_and_unzip(args.build_dir, url, 'cmake_temp')
-        dest_dir = os.path.join(args.build_dir,'cmake')
-        if os.path.exists(dest_dir):
-            print('deleting %s' % dest_dir)
-            shutil.rmtree(dest_dir)
-        shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
-        url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe'
-        url = url.replace('onnxruntimetestdata', hostname)
-        dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage')
-        os.makedirs(dest_folder,exist_ok=True)
-        subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True)
+    all_downloads_done = False
+    azure_region = get_azure_region()
+
+    try:
+        # Download test data
+        url = get_region_based_url(args.test_data_url, azure_region)
+        print("Starting test data download %s" % url)
+        download_and_unzip(args.build_dir, url, models_folder)
+
+        # On windows download additional data
+        if is_windows():
+            download_additional_data(args.build_dir, azure_region)
+
+        all_downloads_done = True
+
+    except Exception as e:
+        print(e)
+
+    if not all_downloads_done:
+        raise Exception("Downloading test data step failed.")