From 4caf5c9c136756cdf0877b873e983d3fcc22f301 Mon Sep 17 00:00:00 2001 From: Ashwini Khade Date: Mon, 25 Nov 2019 13:08:03 -0800 Subject: [PATCH] add additional test data set for nuget pipeline (#2448) * add SAS token to download internal test data for nuget pipeline * update azure endpoint * fix keyvault download step * fix variable declaration for secret group * fix indentation * fix yaml syntax for variables * fix setting secrets for script * fix env synctax * Fix macos pipeline * attempt to add secrets to windows download data * fix mac and win data download * fix windows data download * update test data set url and location --- .../c-api-packaging-pipelines.yml | 1 - .../linux-set-variables-and-download.yml | 29 ++++ .../mac-set-variables-and-download.yml | 18 +++ .../set-test-data-variables-step.yml | 11 ++ .../nuget/templates/test_linux.yml | 7 +- .../nuget/templates/test_macos.yml | 7 +- .../nuget/templates/test_win.yml | 15 +- .../templates/win-download-test-data.yml | 18 +++ .../windows-build-tools-setup-steps.yml | 3 + tools/ci_build/github/download_test_data.py | 144 +++++++++++++----- 10 files changed, 202 insertions(+), 51 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/mac-set-variables-and-download.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/set-test-data-variables-step.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 498f3a607b..2671762fc3 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -56,7 +56,6 @@ jobs: pool: vmImage: 'macOS-10.13' steps: - - template: templates/mac-set-variables-and-download.yml - template: templates/set-version-number-variables-step.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml new file mode 100644 index 0000000000..0ebff16246 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml @@ -0,0 +1,29 @@ +steps: +- task: CmdLine@2 + displayName: 'Clean untagged docker images' + inputs: + script: | + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker container prune -f + docker image prune -f + workingDirectory: $(Build.BinariesDirectory) + continueOnError: true + condition: always() + +- task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + +- task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + env: + Test_Data_Download_Key: $(ort-internal-test-data-sas) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/mac-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/mac-set-variables-and-download.yml new file mode 100644 index 0000000000..20127f44d7 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/mac-set-variables-and-download.yml @@ -0,0 +1,18 @@ +steps: +- task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-mac' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + +- task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --azure_region centralus --build_dir $(Build.BinariesDirectory) + pythonInterpreter: '/usr/local/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + env: + Test_Data_Download_Key: $(ort-internal-test-data-sas) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/set-test-data-variables-step.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/set-test-data-variables-step.yml new file mode 100644 index 0000000000..cf35e6b56d --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/set-test-data-variables-step.yml @@ -0,0 +1,11 @@ +# sets variables $(TestDataUrl) + +parameters: + TestDataUrl: https://ortinternaltestdata.blob.core.windows.net/models/20191122.zip + +steps: +- task: CmdLine@1 + displayName: 'Set TestDataUrl variable' + inputs: + filename: echo + arguments: '##vso[task.setvariable variable=TestDataUrl;]${{parameters.TestDataUrl}}' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 0c423e61d3..1071e5937a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -11,9 +11,12 @@ jobs: - NuGet_Packaging condition: succeeded() variables: - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + - group: ORT_TEST_DATA_SAS + - name: OnnxRuntimeBuildDirectory + value: '$(Build.BinariesDirectory)' steps: - - template: ../../templates/linux-set-variables-and-download.yml + - template: set-test-data-variables-step.yml + - template: linux-set-variables-and-download.yml - task: DownloadPipelineArtifact@0 displayName: 'Download Signed NuGet' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index cd5f4d7505..347bc78ac1 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -10,9 +10,12 @@ jobs: - NuGet_Packaging condition: succeeded() variables: - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + - group: ORT_TEST_DATA_SAS + - name: OnnxRuntimeBuildDirectory + value: '$(Build.BinariesDirectory)' steps: - - template: ../../templates/mac-set-variables-and-download.yml + - template: set-test-data-variables-step.yml + - template: mac-set-variables-and-download.yml - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - Signed NuGet' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index be9ca6ddd1..3d506a1ab0 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -10,14 +10,19 @@ jobs: - NuGet_Packaging condition: succeeded() variables: - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + - group: ORT_TEST_DATA_SAS + - name: OnnxRuntimeBuildDirectory + value: '$(Build.BinariesDirectory)' + steps: - - template: ../../templates/set-test-data-variables-step.yml + - template: set-test-data-variables-step.yml - template: ../../templates/windows-build-tools-setup-steps.yml parameters: EnvSetupScript: 'setup_env.bat' buildArch: 'x64' setVcvars: true + DoDataDownload: false + - template: win-download-test-data.yml - task: BatchScript@1 displayName: 'Setup VS2017 env vars' @@ -34,12 +39,6 @@ jobs: - template: get-nuget-package-version-as-variable.yml parameters: packageFolder: '$(Build.BinariesDirectory)\nuget-artifact' - - - script: | - @echo "Running download_test_data.py" - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) - - displayName: 'Download Test Data' - script: | @echo "Running Runtest.bat" diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml new file mode 100644 index 0000000000..f5d5790c56 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml @@ -0,0 +1,18 @@ +# Assumes AZCopy and Python download is already done +steps: +- task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' + workingDirectory: $(Build.BinariesDirectory) + env: + Test_Data_Download_Key: $(ort-internal-test-data-sas) + +- task: CmdLine@1 + continueOnError: true + displayName: 'Run OpenCPPCoverage installer' + inputs: + filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe' + arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index 6002951ed3..ec13ca83b2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -2,6 +2,7 @@ parameters: EnvSetupScript: '' + DoDataDownload: true steps: - task: NuGetToolInstaller@0 displayName: Use Nuget 4.9 @@ -53,6 +54,7 @@ steps: - task: PythonScript@0 displayName: 'Download test data' + condition: ${{parameters.DoDataDownload}} inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) @@ -62,6 +64,7 @@ steps: - task: CmdLine@1 continueOnError: true displayName: 'Run OpenCPPCoverage installer' + condition: ${{parameters.DoDataDownload}} inputs: filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe' arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"' diff --git a/tools/ci_build/github/download_test_data.py b/tools/ci_build/github/download_test_data.py index 06347a61ea..bb8fea8adf 100755 --- a/tools/ci_build/github/download_test_data.py +++ b/tools/ci_build/github/download_test_data.py @@ -7,16 +7,39 @@ import sys import shutil import argparse from urllib.parse import urlparse +from urllib.parse import urljoin +from urllib.parse import urlsplit + +# Hardcoded map of storage account to azure region endpoint +storage_account_to_endpoint_map = { + 'onnxruntimetestdata.blob.core.windows.net' : { + 'centralus' : 'onnxruntimetestdata.blob.core.windows.net', + 'northcentralus' : 'onnxruntimetestdata2.blob.core.windows.net' + }, + 'ortinternaltestdata' : { + 'centralus' : 'ortinternaltestdata.blob.core.windows.net' + } +} def is_windows(): return sys.platform.startswith("win") def get_azure_region(): - req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01') - req.add_header('Metadata', 'true') - body = urllib.request.urlopen(req).read() - body = json.loads(body.decode('utf-8')) - return body['compute']['location'] + # Try to get azure location for this machine + if args.azure_region is not None: + return args.azure_region + else: + azure_region = None + try: + req = urllib.request.Request('http://169.254.169.254/metadata/instance?api-version=2018-10-01') + req.add_header('Metadata', 'true') + body = urllib.request.urlopen(req).read() + body = json.loads(body.decode('utf-8')) + azure_region = body['compute']['location'] + print("This VM is in azure region: %s" % azure_region) + except: + print("Failed to get azure region for this machine.") + return azure_region def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime Data Downloader.") @@ -27,25 +50,45 @@ def parse_arguments(): return parser.parse_args() -def get_server_hostname(azure_location): - if azure_location is None: - #should be northcentralus or centralus - azure_location = get_azure_region() - print("This VM is in azure location: %s" % azure_location) - if azure_location == 'centralus': - hostname = 'onnxruntimetestdata' - elif azure_location == 'northcentralus': - hostname = 'onnxruntimetestdata2' +def get_endpoint_by_region(current_endpoint, azure_location): + # Find storage account closest to the current region + # TODO : Ideally we should replicate same storage account to multiple regions + # and choose the endpoint accordigly instead of the current way where + # we maintain separate storage accounts per region. + if current_endpoint in storage_account_to_endpoint_map: + endpoint_to_region_map = storage_account_to_endpoint_map[current_endpoint] + if azure_location in endpoint_to_region_map: + return endpoint_to_region_map[azure_location] + else: + print('warning: no local data cache for azure region %s' % azure_location) + return current_endpoint else: print('warning: no local data cache for azure region %s' % azure_location) - hostname = 'onnxruntimetestdata2' - return hostname + return current_endpoint + return current_endpoint +def get_region_based_url(url, azure_location): + current_endpoint = urlsplit(url).netloc + endpoint_by_region = get_endpoint_by_region(current_endpoint, args.azure_region) + url = url.replace(current_endpoint, endpoint_by_region) + print("url changed to %s" % url) + return url -def download_and_unzip(build_dir, url, dest_folder): - print("Downloading %s" % url) +def download_and_unzip(build_dir, url, dest_folder, use_token = True): dest_folder = os.path.join(build_dir, dest_folder) - subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', url, build_dir],check=True) + # attach the SAS token to the url. Note DO NOT print the url with the token in any logs. + token = os.environ.get('Test_Data_Download_Key') + if use_token: + url_with_token = urljoin(url, token) + else: + url_with_token = url + + # Download data using AZCopy tool + try: + subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', '--recursive', url_with_token, build_dir],check=True) + except: + raise Exception("Downloading data failed. Source: " + url + " Destination: " + build_dir) + os.makedirs(dest_folder,exist_ok=True) local_file_name = os.path.join(build_dir, os.path.basename(urlparse(url).path)) if is_windows(): @@ -61,30 +104,55 @@ def download_and_unzip(build_dir, url, dest_folder): subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) os.unlink(local_file_name) +def download_additional_data(build_dir, azure_region): + additional_data_url = 'https://onnxruntimetestdata.blob.core.windows.net/models/' + url = get_region_based_url(args.test_data_url, azure_region) + + cmake_url = urljoin(additional_data_url, 'cmake-3.15.1-win64-x64.zip') + print("Starting download for cmake : " + cmake_url) + download_and_unzip(build_dir, cmake_url, 'cmake_temp', False) + dest_dir = os.path.join(build_dir,'cmake') + if os.path.exists(dest_dir): + print('deleting %s' % dest_dir) + shutil.rmtree(dest_dir) + shutil.move(os.path.join(build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) + + # Download OpenCPPCoverageSetup.exe + opencpp_url = urljoin(additional_data_url, 'OpenCppCoverageSetup-x64-0.9.7.0.exe') + print("Starting download for opencppcoverage " + opencpp_url) + dest_folder = os.path.join(build_dir, 'installer','opencppcoverage') + os.makedirs(dest_folder,exist_ok=True) + subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', opencpp_url, os.path.join(dest_folder,'installer.exe')],check=True) + args = parse_arguments() models_folder = 'models' + if args.edge_device: dest_folder = os.path.join(args.build_dir, models_folder) #For edge device, the model zip file is persist at /mnt/ubuntu/tmp/model.zip local_file_name = '/mnt/ubuntu/tmp/model.zip' if os.path.exists(local_file_name): - subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) + subprocess.run(['unzip','-qd', dest_folder ,local_file_name], check=True) + else: + raise Exception(local_file_name + " does not exist on edge device. Downloading test data step failed.") else: - hostname = get_server_hostname(args.azure_region) - url = args.test_data_url.replace('onnxruntimetestdata', hostname) - print('data url=%s' % url) - download_and_unzip(args.build_dir, url, models_folder) - if is_windows(): - url = 'https://onnxruntimetestdata.blob.core.windows.net/models/cmake-3.15.1-win64-x64.zip' - url = url.replace('onnxruntimetestdata', hostname) - download_and_unzip(args.build_dir, url, 'cmake_temp') - dest_dir = os.path.join(args.build_dir,'cmake') - if os.path.exists(dest_dir): - print('deleting %s' % dest_dir) - shutil.rmtree(dest_dir) - shutil.move(os.path.join(args.build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir) - url = 'https://onnxruntimetestdata.blob.core.windows.net/models/OpenCppCoverageSetup-x64-0.9.7.0.exe' - url = url.replace('onnxruntimetestdata', hostname) - dest_folder = os.path.join(args.build_dir, 'installer','opencppcoverage') - os.makedirs(dest_folder,exist_ok=True) - subprocess.run([os.path.join(args.build_dir,'azcopy'),'cp', '--log-level','ERROR', url, os.path.join(dest_folder,'installer.exe')],check=True) + all_downloads_done = False + azure_region = get_azure_region() + + try: + # Download test data + url = get_region_based_url(args.test_data_url, azure_region) + print("Starting test data download %s" % url) + download_and_unzip(args.build_dir, url, models_folder) + + # On windows download additional data + if is_windows(): + download_additional_data(args.build_dir, azure_region) + + all_downloads_done = True + + except Exception as e: + print(e) + + if not all_downloads_done: + raise Exception("Downloading test data step failed.") \ No newline at end of file