diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml index c3d5bebe99..b34d9f791d 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml @@ -77,26 +77,15 @@ stages: inputs: ArtifactName: onnxruntime_training_cpu - - script: | - python3 -m pip install azure-storage-blob==2.1.0 - displayName: 'python3 -m pip install azure-storage-blob==2.1.0' - timeoutInMinutes: 20 - - - task: AzureCLI@2 + - task: CmdLine@2 + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + displayName: 'Upload wheel' inputs: - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | + script: | files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ echo ${files[0]} && \ tools/ci_build/upload_python_package_to_azure_storage.py \ - --python_wheel_path ${files[0]} \ - --account_name onnxruntimepackages \ - --account_key $(orttrainingpackagestorageaccountkey) \ - --container_name '$web' - condition: succeededOrFailed() - displayName: + --python_wheel_path ${files[0]} - template: templates/component-governance-component-detection-steps.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 0d80bf1bcc..14030d241c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -597,14 +597,10 @@ stages: ArtifactName: onnxruntime_rocm - script: | - python3 -m pip install azure-storage-blob==2.1.0 files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ echo ${files[0]} && \ python3 tools/ci_build/upload_python_package_to_azure_storage.py \ - --python_wheel_path ${files[0]} \ - --account_name onnxruntimepackages \ - --account_key $(orttrainingpackagestorageaccountkey) \ - --container_name '$web' + --python_wheel_path ${files[0]} condition: and(succeeded(), eq(variables['DRY_RUN'], '0')) displayName: 'Upload Rocm wheel to release repository' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml index 6cc5fc6305..85dd578255 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml @@ -206,22 +206,16 @@ stages: inputs: ArtifactName: onnxruntime_gpu - - task: AzureCLI@2 + - task: CmdLine@2 + displayName: 'Upload wheel' condition: and(succeeded(), eq(variables['UploadWheel'], 'yes')) inputs: - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - python3 -m pip install azure-storage-blob==2.1.0 + script: | + set -e -x files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ echo ${files[0]} && \ python3 tools/ci_build/upload_python_package_to_azure_storage.py \ - --python_wheel_path ${files[0]} \ - --account_name onnxruntimepackages \ - --account_key $(orttrainingpackagestorageaccountkey) \ - --container_name '$web' - displayName: + --python_wheel_path ${files[0]} - template: component-governance-component-detection-steps.yml parameters: diff --git a/tools/ci_build/upload_python_package_to_azure_storage.py b/tools/ci_build/upload_python_package_to_azure_storage.py index da7e4a8ea5..a0b3ad5836 100755 --- a/tools/ci_build/upload_python_package_to_azure_storage.py +++ b/tools/ci_build/upload_python_package_to_azure_storage.py @@ -5,7 +5,10 @@ import os import argparse import warnings -from azure.storage.blob import BlockBlobService, ContentSettings +import subprocess +import logging + +log = logging.getLogger("Build") def parse_nightly_and_local_version_from_whl_name(blob_name): @@ -21,14 +24,14 @@ def parse_nightly_and_local_version_from_whl_name(blob_name): return night_build, blob_name[start:end] -def upload_whl(python_wheel_path, account_name, account_key, container_name): - block_blob_service = BlockBlobService( - account_name=account_name, - account_key=account_key - ) +def run_subprocess(args, cwd=None): + log.debug("Running subprocess in '{0}'\n{1}".format(cwd or os.getcwd(), args)) + return subprocess.run(args, cwd=cwd, check=True) + +def upload_whl(python_wheel_path): blob_name = os.path.basename(python_wheel_path) - block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path) + run_subprocess(['azcopy', 'cp', python_wheel_path, 'https://onnxruntimepackages.blob.core.windows.net/$web/']) nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name) if local_version: @@ -38,7 +41,8 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name): download_path_to_html = "./onnxruntime_{}.html".format(nightly_build) - block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html) + run_subprocess(['azcopy', 'cp', 'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name, + download_path_to_html]) blob_name_plus_replaced = blob_name.replace('+', '%2B') with open(download_path_to_html) as f: @@ -54,25 +58,16 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name): f.write("%s\n" % item) else: warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html)) - - content_settings = ContentSettings(content_type='text/html') - block_blob_service.create_blob_from_path( - container_name, - html_blob_name, - download_path_to_html, - content_settings=content_settings) + run_subprocess(['azcopy', 'cp', download_path_to_html, + 'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name, + '--content-type', 'text/html', '--overwrite', 'true']) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Upload python whl to azure storage.") parser.add_argument("--python_wheel_path", type=str, help="path to python wheel") - parser.add_argument("--account_name", type=str, help="account name") - parser.add_argument("--account_key", type=str, help="account key") - parser.add_argument("--container_name", type=str, help="container name") - # TODO: figure out a way to secure args.account_key to prevent later code changes - # that may accidentally print out it to the console. args = parser.parse_args() - upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name) + upload_whl(args.python_wheel_path)