mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Patch Release 1.11.1 cherry pick (#11255)
* Update tools/ci_build/upload_python_package_to_azure_storage.py to not use the azure blob storage python package (#11114) * Fix the rocm packaging pipeline package upload problem (#11174) In #11114 , I changed the script to use azcopy instead of azure blob storage's python APIs. However, it doesn't work for the AMD rocm pipeline, because: 1. The machines do not have azcopy installed 2. The machines are not in Azure, so they don't have Azure managed identity. So they still need to use SAS. Therefore in this PR I get the old python file back, but only use it in the AMD pipeline. * Scoped GIL release in run_with_iobinding (#11248) * [js/web] disable test_tan temorarily (#11048) * [js/web] fix output type mapping (#11049) Co-authored-by: Changming Sun <chasun@microsoft.com> Co-authored-by: Dmitri Smirnov <yuslepukhin@users.noreply.github.com> Co-authored-by: Yulong Wang <7679871+fs-eire@users.noreply.github.com>
This commit is contained in:
Chi Lo
GitHub
parent
e92e751151
commit
8f5c81bb05
9 changed files with 410 additions and 394 deletions
|
|
@ -176,9 +176,9 @@ const tensorDataTypeEnumToString = (typeProto: DataType): Tensor.Type => {
|
|||
case DataType.string:
|
||||
return 'string';
|
||||
case DataType.int64:
|
||||
return 'int32';
|
||||
return 'int64';
|
||||
case DataType.uint64:
|
||||
return 'uint32';
|
||||
return 'uint64';
|
||||
|
||||
default:
|
||||
throw new Error(`unsupported data type: ${typeProto}`);
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@
|
|||
// "test_slice_start_out_of_bounds", // tensor shape of 0
|
||||
"test_squeeze",
|
||||
"test_tan_example",
|
||||
"test_tan",
|
||||
//"test_tan", // incorrect result in Chrome v100... investigating
|
||||
"test_tanh_example",
|
||||
"test_tanh",
|
||||
"test_tile",
|
||||
|
|
|
|||
|
|
@ -1497,6 +1497,8 @@ including arg name, arg type (contains both type and shape).)pbdoc")
|
|||
py::return_value_policy::reference_internal)
|
||||
.def("run_with_iobinding", [](PyInferenceSession* sess, SessionIOBinding& io_binding, RunOptions* run_options = nullptr) -> void {
|
||||
Status status;
|
||||
// release GIL to allow multiple python threads to invoke Run() in parallel.
|
||||
py::gil_scoped_release release;
|
||||
if (!run_options)
|
||||
status = sess->GetSessionHandle()->Run(*io_binding.Get());
|
||||
else
|
||||
|
|
|
|||
|
|
@ -77,26 +77,15 @@ stages:
|
|||
inputs:
|
||||
ArtifactName: onnxruntime_training_cpu
|
||||
|
||||
- script: |
|
||||
python3 -m pip install azure-storage-blob==2.1.0
|
||||
displayName: 'python3 -m pip install azure-storage-blob==2.1.0'
|
||||
timeoutInMinutes: 20
|
||||
|
||||
- task: AzureCLI@2
|
||||
- task: CmdLine@2
|
||||
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
|
||||
displayName: 'Upload wheel'
|
||||
inputs:
|
||||
azureSubscription: 'AIInfraBuildOnnxRuntimeOSS'
|
||||
scriptType: 'bash'
|
||||
scriptLocation: 'inlineScript'
|
||||
inlineScript: |
|
||||
script: |
|
||||
files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
|
||||
echo ${files[0]} && \
|
||||
tools/ci_build/upload_python_package_to_azure_storage.py \
|
||||
--python_wheel_path ${files[0]} \
|
||||
--account_name onnxruntimepackages \
|
||||
--account_key $(orttrainingpackagestorageaccountkey) \
|
||||
--container_name '$web'
|
||||
condition: succeededOrFailed()
|
||||
displayName:
|
||||
--python_wheel_path ${files[0]}
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
|
|
|
|||
|
|
@ -1,15 +1,304 @@
|
|||
trigger: none
|
||||
|
||||
stages:
|
||||
- template: templates/py-packaging-stage.yml
|
||||
parameters:
|
||||
build_py_parameters: --enable_training
|
||||
enable_linux_cpu: false
|
||||
enable_linux_gpu: false
|
||||
enable_linux_gpu_training_cu102: false
|
||||
enable_linux_gpu_training_cu111: false
|
||||
enable_linux_rocm_training: true
|
||||
enable_windows_cpu: false
|
||||
enable_windows_gpu: false
|
||||
enable_mac_cpu: false
|
||||
enable_linux_arm: false
|
||||
- stage: Python_Packaging
|
||||
jobs:
|
||||
- job: ROCm_build_environment
|
||||
displayName: 'Construct ROCm wheels environment'
|
||||
timeoutInMinutes: 300
|
||||
workspace:
|
||||
clean: all
|
||||
pool: AMD-GPU
|
||||
steps:
|
||||
- template: templates/ get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: >-
|
||||
--build-arg TORCH_VERSION=1.11.0
|
||||
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
|
||||
--build-arg BUILD_UID=$(id -u)
|
||||
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
|
||||
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
|
||||
--build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
|
||||
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
|
||||
Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.3.1
|
||||
- template: templates/ get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: >-
|
||||
--build-arg TORCH_VERSION=1.11.0
|
||||
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
|
||||
--build-arg BUILD_UID=$(id -u)
|
||||
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
|
||||
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
|
||||
--build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
|
||||
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
|
||||
Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm5.0.1
|
||||
|
||||
- job: ROCM_training_wheels
|
||||
timeoutInMinutes: 180
|
||||
displayName: 'Build ROCm wheels (inside container)'
|
||||
workspace:
|
||||
clean: all
|
||||
pool: AMD-GPU
|
||||
dependsOn:
|
||||
- ROCm_build_environment
|
||||
strategy:
|
||||
matrix:
|
||||
Python37 Torch1100 Rocm431:
|
||||
PythonVersion: '3.7'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.3.1'
|
||||
Python38 Torch1100 Rocm431:
|
||||
PythonVersion: '3.8'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.3.1'
|
||||
Python39 Torch1100 Rocm431:
|
||||
PythonVersion: '3.9'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.3.1'
|
||||
Python37 Torch1100 Rocm501:
|
||||
PythonVersion: '3.7'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '5.0.1'
|
||||
Python38 Torch1100 Rocm501:
|
||||
PythonVersion: '3.8'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '5.0.1'
|
||||
Python39 Torch1100 Rocm501:
|
||||
PythonVersion: '3.9'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '5.0.1'
|
||||
steps:
|
||||
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- template: templates/ set-python-manylinux-variables-step.yml
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
docker run --rm \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
-e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/build.py \
|
||||
--config Release \
|
||||
--use_rocm \
|
||||
--rocm_version=$(RocmVersion) \
|
||||
--rocm_home=/opt/rocm \
|
||||
--nccl_home=/opt/rocm \
|
||||
--update \
|
||||
--parallel \
|
||||
--build_dir /build \
|
||||
--build \
|
||||
--build_wheel \
|
||||
--skip_tests \
|
||||
--enable_training
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
displayName: 'Build onnxruntime (in container)'
|
||||
|
||||
- script: |-
|
||||
python3 orttraining/tools/ci_test/download_azure_blob_archive.py \
|
||||
--azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
|
||||
--target_dir $(Build.SourcesDirectory)/training_e2e_test_data \
|
||||
--archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9
|
||||
displayName: 'Download onnxruntime_training_data.zip data'
|
||||
|
||||
- script: |-
|
||||
echo "Tests will run using HIP_VISIBLES_DEVICES=$HIP_VISIBLE_DEVICES"
|
||||
video_gid=$(getent group | awk '/video/ {split($0,a,":"); print(a[3])}')
|
||||
echo "Found video_gid=$video_gid; attempting to set as pipeline variable"
|
||||
echo "##vso[task.setvariable variable=video]$video_gid"
|
||||
render_gid=$(getent group | awk '/render/ {split($0,a,":"); print(a[3])}')
|
||||
echo "Found render_gid=$render_gid; attempting to set as pipeline variable"
|
||||
echo "##vso[task.setvariable variable=render]$render_gid"
|
||||
displayName: 'Find video and render gid to be mapped into container'
|
||||
|
||||
- script: |-
|
||||
echo "video=$video"
|
||||
echo "render=$render"
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /build/Release \
|
||||
--entrypoint /bin/bash \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh
|
||||
displayName: 'Run onnxruntime unit tests (in container)'
|
||||
|
||||
- script: |-
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
orttraining/tools/ci_test/run_batch_size_test.py \
|
||||
--binary_dir /build/Release \
|
||||
--model_root training_e2e_test_data/models \
|
||||
--gpu_sku MI100_32G
|
||||
displayName: 'Run C++ BERT-L batch size test (in container)'
|
||||
condition: succeededOrFailed() # ensure all tests are run
|
||||
|
||||
- script: |-
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
orttraining/tools/ci_test/run_bert_perf_test.py \
|
||||
--binary_dir /build/Release \
|
||||
--model_root training_e2e_test_data/models \
|
||||
--training_data_root training_e2e_test_data/data \
|
||||
--gpu_sku MI100_32G
|
||||
displayName: 'Run C++ BERT-L performance test (in container)'
|
||||
condition: succeededOrFailed() # ensure all tests are run
|
||||
|
||||
- script: |-
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
orttraining/tools/ci_test/run_convergence_test.py \
|
||||
--binary_dir /build/Release \
|
||||
--model_root training_e2e_test_data/models \
|
||||
--training_data_root training_e2e_test_data/data \
|
||||
--gpu_sku MI100_32G
|
||||
displayName: 'Run C++ BERT-L convergence test (in container)'
|
||||
condition: succeededOrFailed() # ensure all tests are run
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)'
|
||||
Contents: 'Release/dist/*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Build Python Documentation'
|
||||
condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9
|
||||
inputs:
|
||||
script: |
|
||||
mkdir -p $HOME/.onnx
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--entrypoint /bin/bash \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
-e PythonManylinuxDir=$(PythonManylinuxdir) \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)'
|
||||
condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)/docs/training/html'
|
||||
Contents: '**'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)/training_html_doc'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Upload Rocm wheel as build artifact'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime_rocm
|
||||
|
||||
- script: |
|
||||
python3 -m pip install azure-storage-blob==2.1.0
|
||||
files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
|
||||
echo ${files[0]} && \
|
||||
python3 tools/ci_build/upload_python_package_to_azure_storage_with_python.py \
|
||||
--python_wheel_path ${files[0]} \
|
||||
--account_name onnxruntimepackages \
|
||||
--account_key $(orttrainingpackagestorageaccountkey) \
|
||||
--container_name '$web'
|
||||
condition: and(succeeded(), eq(variables['DRY_RUN'], '0'))
|
||||
displayName: 'Upload Rocm wheel to release repository'
|
||||
|
||||
- template: templates/ component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
||||
- template: templates/ clean-agent-build-directory-step.yml
|
||||
|
|
|
|||
|
|
@ -25,11 +25,6 @@ parameters:
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: enable_linux_rocm_training
|
||||
displayName: 'Whether Linux ROCM package is built.'
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: enable_windows_cpu
|
||||
displayName: 'Whether Windows CPU package is built.'
|
||||
type: boolean
|
||||
|
|
@ -284,332 +279,6 @@ stages:
|
|||
|
||||
- template: clean-agent-build-directory-step.yml
|
||||
|
||||
- ${{ if eq(parameters.enable_linux_rocm_training, true) }}:
|
||||
- job: ROCm_build_environment
|
||||
displayName: 'Construct ROCm wheels environment'
|
||||
timeoutInMinutes: 300
|
||||
workspace:
|
||||
clean: all
|
||||
pool: AMD-GPU
|
||||
steps:
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: >-
|
||||
--build-arg TORCH_VERSION=1.10.0
|
||||
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
|
||||
--build-arg BUILD_UID=$(id -u)
|
||||
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
|
||||
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
|
||||
--build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
|
||||
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
|
||||
Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.2
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: >-
|
||||
--build-arg TORCH_VERSION=1.10.0
|
||||
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
|
||||
--build-arg BUILD_UID=$(id -u)
|
||||
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
|
||||
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
|
||||
--build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
|
||||
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
|
||||
Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.3.1
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: >-
|
||||
--build-arg TORCH_VERSION=1.10.0
|
||||
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
|
||||
--build-arg BUILD_UID=$(id -u)
|
||||
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
|
||||
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
|
||||
--build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
|
||||
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
|
||||
Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm5.0.1
|
||||
|
||||
- job: ROCM_training_wheels
|
||||
timeoutInMinutes: 180
|
||||
displayName: 'Build ROCm wheels (inside container)'
|
||||
workspace:
|
||||
clean: all
|
||||
pool: AMD-GPU
|
||||
dependsOn:
|
||||
- ROCm_build_environment
|
||||
strategy:
|
||||
matrix:
|
||||
Python37 Torch1100 Rocm42:
|
||||
PythonVersion: '3.7'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.2'
|
||||
Python38 Torch1100 Rocm42:
|
||||
PythonVersion: '3.8'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.2'
|
||||
Python39 Torch1100 Rocm42:
|
||||
PythonVersion: '3.9'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.2'
|
||||
Python37 Torch1100 Rocm431:
|
||||
PythonVersion: '3.7'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.3.1'
|
||||
Python38 Torch1100 Rocm431:
|
||||
PythonVersion: '3.8'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.3.1'
|
||||
Python39 Torch1100 Rocm431:
|
||||
PythonVersion: '3.9'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '4.3.1'
|
||||
Python37 Torch1100 Rocm501:
|
||||
PythonVersion: '3.7'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '5.0.1'
|
||||
Python38 Torch1100 Rocm501:
|
||||
PythonVersion: '3.8'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '5.0.1'
|
||||
Python39 Torch1100 Rocm501:
|
||||
PythonVersion: '3.9'
|
||||
TorchVersion: '1.10.0'
|
||||
RocmVersion: '5.0.1'
|
||||
steps:
|
||||
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- template: set-python-manylinux-variables-step.yml
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
docker run --rm \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
-e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/build.py \
|
||||
--config Release \
|
||||
--use_rocm \
|
||||
--rocm_version=$(RocmVersion) \
|
||||
--rocm_home=/opt/rocm \
|
||||
--nccl_home=/opt/rocm \
|
||||
--update \
|
||||
--parallel \
|
||||
--build_dir /build \
|
||||
--build \
|
||||
--build_wheel \
|
||||
--skip_tests \
|
||||
${{ parameters.build_py_parameters }}
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
displayName: 'Build onnxruntime (in container)'
|
||||
|
||||
- script: |-
|
||||
python3 orttraining/tools/ci_test/download_azure_blob_archive.py \
|
||||
--azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
|
||||
--target_dir $(Build.SourcesDirectory)/training_e2e_test_data \
|
||||
--archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9
|
||||
displayName: 'Download onnxruntime_training_data.zip data'
|
||||
|
||||
- script: |-
|
||||
echo "Tests will run using HIP_VISIBLES_DEVICES=$HIP_VISIBLE_DEVICES"
|
||||
video_gid=$(getent group | awk '/video/ {split($0,a,":"); print(a[3])}')
|
||||
echo "Found video_gid=$video_gid; attempting to set as pipeline variable"
|
||||
echo "##vso[task.setvariable variable=video]$video_gid"
|
||||
render_gid=$(getent group | awk '/render/ {split($0,a,":"); print(a[3])}')
|
||||
echo "Found render_gid=$render_gid; attempting to set as pipeline variable"
|
||||
echo "##vso[task.setvariable variable=render]$render_gid"
|
||||
displayName: 'Find video and render gid to be mapped into container'
|
||||
|
||||
- script: |-
|
||||
echo "video=$video"
|
||||
echo "render=$render"
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /build/Release \
|
||||
--entrypoint /bin/bash \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh
|
||||
displayName: 'Run onnxruntime unit tests (in container)'
|
||||
|
||||
- script: |-
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
orttraining/tools/ci_test/run_batch_size_test.py \
|
||||
--binary_dir /build/Release \
|
||||
--model_root training_e2e_test_data/models \
|
||||
--gpu_sku MI100_32G
|
||||
displayName: 'Run C++ BERT-L batch size test (in container)'
|
||||
condition: succeededOrFailed() # ensure all tests are run
|
||||
|
||||
- script: |-
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
orttraining/tools/ci_test/run_bert_perf_test.py \
|
||||
--binary_dir /build/Release \
|
||||
--model_root training_e2e_test_data/models \
|
||||
--training_data_root training_e2e_test_data/data \
|
||||
--gpu_sku MI100_32G
|
||||
displayName: 'Run C++ BERT-L performance test (in container)'
|
||||
condition: succeededOrFailed() # ensure all tests are run
|
||||
|
||||
- script: |-
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--workdir /onnxruntime_src \
|
||||
--entrypoint $(PythonManylinuxDir)/bin/python3 \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
--user onnxruntimedev \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
orttraining/tools/ci_test/run_convergence_test.py \
|
||||
--binary_dir /build/Release \
|
||||
--model_root training_e2e_test_data/models \
|
||||
--training_data_root training_e2e_test_data/data \
|
||||
--gpu_sku MI100_32G
|
||||
displayName: 'Run C++ BERT-L convergence test (in container)'
|
||||
condition: succeededOrFailed() # ensure all tests are run
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)'
|
||||
Contents: 'Release/dist/*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Build Python Documentation'
|
||||
condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9
|
||||
inputs:
|
||||
script: |
|
||||
mkdir -p $HOME/.onnx
|
||||
docker run --rm \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--group-add $(video) \
|
||||
--group-add $(render) \
|
||||
--privileged \
|
||||
--ipc=host \
|
||||
--network=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--entrypoint /bin/bash \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
-e PythonManylinuxDir=$(PythonManylinuxdir) \
|
||||
onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
|
||||
/onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)'
|
||||
condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)/docs/training/html'
|
||||
Contents: '**'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)/training_html_doc'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Upload Rocm wheel as build artifact'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime_rocm
|
||||
|
||||
- script: |
|
||||
python3 -m pip install azure-storage-blob==2.1.0
|
||||
files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
|
||||
echo ${files[0]} && \
|
||||
python3 tools/ci_build/upload_python_package_to_azure_storage.py \
|
||||
--python_wheel_path ${files[0]} \
|
||||
--account_name onnxruntimepackages \
|
||||
--account_key $(orttrainingpackagestorageaccountkey) \
|
||||
--container_name '$web'
|
||||
condition: and(succeeded(), eq(variables['DRY_RUN'], '0'))
|
||||
displayName: 'Upload Rocm wheel to release repository'
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
||||
- template: clean-agent-build-directory-step.yml
|
||||
|
||||
- ${{ if eq(parameters.enable_windows_cpu, true) }}:
|
||||
- job: Windows_py_Wheels
|
||||
pool: 'Win-CPU-2021'
|
||||
|
|
|
|||
|
|
@ -206,22 +206,16 @@ stages:
|
|||
inputs:
|
||||
ArtifactName: onnxruntime_gpu
|
||||
|
||||
- task: AzureCLI@2
|
||||
- task: CmdLine@2
|
||||
displayName: 'Upload wheel'
|
||||
condition: and(succeeded(), eq(variables['UploadWheel'], 'yes'))
|
||||
inputs:
|
||||
azureSubscription: 'AIInfraBuildOnnxRuntimeOSS'
|
||||
scriptType: 'bash'
|
||||
scriptLocation: 'inlineScript'
|
||||
inlineScript: |
|
||||
python3 -m pip install azure-storage-blob==2.1.0
|
||||
script: |
|
||||
set -e -x
|
||||
files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
|
||||
echo ${files[0]} && \
|
||||
python3 tools/ci_build/upload_python_package_to_azure_storage.py \
|
||||
--python_wheel_path ${files[0]} \
|
||||
--account_name onnxruntimepackages \
|
||||
--account_key $(orttrainingpackagestorageaccountkey) \
|
||||
--container_name '$web'
|
||||
displayName:
|
||||
--python_wheel_path ${files[0]}
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@
|
|||
import os
|
||||
import argparse
|
||||
import warnings
|
||||
from azure.storage.blob import BlockBlobService, ContentSettings
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("Build")
|
||||
|
||||
|
||||
def parse_nightly_and_local_version_from_whl_name(blob_name):
|
||||
|
|
@ -21,14 +24,14 @@ def parse_nightly_and_local_version_from_whl_name(blob_name):
|
|||
return night_build, blob_name[start:end]
|
||||
|
||||
|
||||
def upload_whl(python_wheel_path, account_name, account_key, container_name):
|
||||
block_blob_service = BlockBlobService(
|
||||
account_name=account_name,
|
||||
account_key=account_key
|
||||
)
|
||||
def run_subprocess(args, cwd=None):
|
||||
log.debug("Running subprocess in '{0}'\n{1}".format(cwd or os.getcwd(), args))
|
||||
return subprocess.run(args, cwd=cwd, check=True)
|
||||
|
||||
|
||||
def upload_whl(python_wheel_path):
|
||||
blob_name = os.path.basename(python_wheel_path)
|
||||
block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path)
|
||||
run_subprocess(['azcopy', 'cp', python_wheel_path, 'https://onnxruntimepackages.blob.core.windows.net/$web/'])
|
||||
|
||||
nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name)
|
||||
if local_version:
|
||||
|
|
@ -38,7 +41,8 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name):
|
|||
|
||||
download_path_to_html = "./onnxruntime_{}.html".format(nightly_build)
|
||||
|
||||
block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html)
|
||||
run_subprocess(['azcopy', 'cp', 'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name,
|
||||
download_path_to_html])
|
||||
|
||||
blob_name_plus_replaced = blob_name.replace('+', '%2B')
|
||||
with open(download_path_to_html) as f:
|
||||
|
|
@ -54,25 +58,16 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name):
|
|||
f.write("%s\n" % item)
|
||||
else:
|
||||
warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html))
|
||||
|
||||
content_settings = ContentSettings(content_type='text/html')
|
||||
block_blob_service.create_blob_from_path(
|
||||
container_name,
|
||||
html_blob_name,
|
||||
download_path_to_html,
|
||||
content_settings=content_settings)
|
||||
run_subprocess(['azcopy', 'cp', download_path_to_html,
|
||||
'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name,
|
||||
'--content-type', 'text/html', '--overwrite', 'true'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Upload python whl to azure storage.")
|
||||
|
||||
parser.add_argument("--python_wheel_path", type=str, help="path to python wheel")
|
||||
parser.add_argument("--account_name", type=str, help="account name")
|
||||
parser.add_argument("--account_key", type=str, help="account key")
|
||||
parser.add_argument("--container_name", type=str, help="container name")
|
||||
|
||||
# TODO: figure out a way to secure args.account_key to prevent later code changes
|
||||
# that may accidentally print out it to the console.
|
||||
args = parser.parse_args()
|
||||
|
||||
upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name)
|
||||
upload_whl(args.python_wheel_path)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import warnings
|
||||
from azure.storage.blob import BlockBlobService, ContentSettings
|
||||
|
||||
|
||||
def parse_nightly_and_local_version_from_whl_name(blob_name):
|
||||
night_build = 'nightly' if blob_name.find(".dev") > 0 else 'stable'
|
||||
|
||||
start = blob_name.find("+")
|
||||
if start == -1:
|
||||
return night_build, None
|
||||
start = start + 1
|
||||
end = blob_name.find("-", start)
|
||||
if end == -1:
|
||||
return night_build, None
|
||||
return night_build, blob_name[start:end]
|
||||
|
||||
|
||||
def upload_whl(python_wheel_path, account_name, account_key, container_name):
|
||||
block_blob_service = BlockBlobService(
|
||||
account_name=account_name,
|
||||
account_key=account_key
|
||||
)
|
||||
|
||||
blob_name = os.path.basename(python_wheel_path)
|
||||
block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path)
|
||||
|
||||
nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name)
|
||||
if local_version:
|
||||
html_blob_name = 'onnxruntime_{}_{}.html'.format(nightly_build, local_version)
|
||||
else:
|
||||
html_blob_name = 'onnxruntime_{}.html'.format(nightly_build)
|
||||
|
||||
download_path_to_html = "./onnxruntime_{}.html".format(nightly_build)
|
||||
|
||||
block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html)
|
||||
|
||||
blob_name_plus_replaced = blob_name.replace('+', '%2B')
|
||||
with open(download_path_to_html) as f:
|
||||
lines = f.read().splitlines()
|
||||
|
||||
new_line = '<a href="{blobname}">{blobname}</a><br>'.format(blobname=blob_name_plus_replaced)
|
||||
if new_line not in lines:
|
||||
lines.append(new_line)
|
||||
lines.sort()
|
||||
|
||||
with open(download_path_to_html, 'w') as f:
|
||||
for item in lines:
|
||||
f.write("%s\n" % item)
|
||||
else:
|
||||
warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html))
|
||||
|
||||
content_settings = ContentSettings(content_type='text/html')
|
||||
block_blob_service.create_blob_from_path(
|
||||
container_name,
|
||||
html_blob_name,
|
||||
download_path_to_html,
|
||||
content_settings=content_settings)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Upload python whl to azure storage.")
|
||||
|
||||
parser.add_argument("--python_wheel_path", type=str, help="path to python wheel")
|
||||
parser.add_argument("--account_name", type=str, help="account name")
|
||||
parser.add_argument("--account_key", type=str, help="account key")
|
||||
parser.add_argument("--container_name", type=str, help="container name")
|
||||
|
||||
# TODO: figure out a way to secure args.account_key to prevent later code changes
|
||||
# that may accidentally print out it to the console.
|
||||
args = parser.parse_args()
|
||||
|
||||
upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name)
|
||||
Loading…
Reference in a new issue