From 8f5c81bb0519bf570bce144238ceb8436509ca78 Mon Sep 17 00:00:00 2001 From: Chi Lo <54722500+chilo-ms@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:28:10 -0700 Subject: [PATCH] Patch Release 1.11.1 cherry pick (#11255) * Update tools/ci_build/upload_python_package_to_azure_storage.py to not use the azure blob storage python package (#11114) * Fix the rocm packaging pipeline package upload problem (#11174) In #11114 , I changed the script to use azcopy instead of azure blob storage's python APIs. However, it doesn't work for the AMD rocm pipeline, because: 1. The machines do not have azcopy installed 2. The machines are not in Azure, so they don't have Azure managed identity. So they still need to use SAS. Therefore in this PR I get the old python file back, but only use it in the AMD pipeline. * Scoped GIL release in run_with_iobinding (#11248) * [js/web] disable test_tan temorarily (#11048) * [js/web] fix output type mapping (#11049) Co-authored-by: Changming Sun Co-authored-by: Dmitri Smirnov Co-authored-by: Yulong Wang <7679871+fs-eire@users.noreply.github.com> --- js/web/lib/wasm/wasm-core-impl.ts | 4 +- js/web/test/suite-test-list.jsonc | 2 +- .../python/onnxruntime_pybind_state.cc | 2 + .../orttraining-py-packaging-pipeline-cpu.yml | 21 +- ...orttraining-py-packaging-pipeline-rocm.yml | 313 ++++++++++++++++- .../templates/py-packaging-stage.yml | 331 ------------------ .../py-packaging-training-cuda-stage.yml | 16 +- .../upload_python_package_to_azure_storage.py | 37 +- ...on_package_to_azure_storage_with_python.py | 78 +++++ 9 files changed, 410 insertions(+), 394 deletions(-) create mode 100644 tools/ci_build/upload_python_package_to_azure_storage_with_python.py diff --git a/js/web/lib/wasm/wasm-core-impl.ts b/js/web/lib/wasm/wasm-core-impl.ts index d1bcb8bbdc..0bccd1cfc3 100644 --- a/js/web/lib/wasm/wasm-core-impl.ts +++ b/js/web/lib/wasm/wasm-core-impl.ts @@ -176,9 +176,9 @@ const tensorDataTypeEnumToString = (typeProto: DataType): Tensor.Type => { case DataType.string: return 'string'; case DataType.int64: - return 'int32'; + return 'int64'; case DataType.uint64: - return 'uint32'; + return 'uint64'; default: throw new Error(`unsupported data type: ${typeProto}`); diff --git a/js/web/test/suite-test-list.jsonc b/js/web/test/suite-test-list.jsonc index 19d3e59274..7ae4504e20 100644 --- a/js/web/test/suite-test-list.jsonc +++ b/js/web/test/suite-test-list.jsonc @@ -207,7 +207,7 @@ // "test_slice_start_out_of_bounds", // tensor shape of 0 "test_squeeze", "test_tan_example", - "test_tan", + //"test_tan", // incorrect result in Chrome v100... investigating "test_tanh_example", "test_tanh", "test_tile", diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index c1f3d99b83..6b6f5a4de9 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1497,6 +1497,8 @@ including arg name, arg type (contains both type and shape).)pbdoc") py::return_value_policy::reference_internal) .def("run_with_iobinding", [](PyInferenceSession* sess, SessionIOBinding& io_binding, RunOptions* run_options = nullptr) -> void { Status status; + // release GIL to allow multiple python threads to invoke Run() in parallel. + py::gil_scoped_release release; if (!run_options) status = sess->GetSessionHandle()->Run(*io_binding.Get()); else diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml index c3d5bebe99..b34d9f791d 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml @@ -77,26 +77,15 @@ stages: inputs: ArtifactName: onnxruntime_training_cpu - - script: | - python3 -m pip install azure-storage-blob==2.1.0 - displayName: 'python3 -m pip install azure-storage-blob==2.1.0' - timeoutInMinutes: 20 - - - task: AzureCLI@2 + - task: CmdLine@2 + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + displayName: 'Upload wheel' inputs: - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | + script: | files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ echo ${files[0]} && \ tools/ci_build/upload_python_package_to_azure_storage.py \ - --python_wheel_path ${files[0]} \ - --account_name onnxruntimepackages \ - --account_key $(orttrainingpackagestorageaccountkey) \ - --container_name '$web' - condition: succeededOrFailed() - displayName: + --python_wheel_path ${files[0]} - template: templates/component-governance-component-detection-steps.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml index 784e916028..942c092e98 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml @@ -1,15 +1,304 @@ trigger: none stages: -- template: templates/py-packaging-stage.yml - parameters: - build_py_parameters: --enable_training - enable_linux_cpu: false - enable_linux_gpu: false - enable_linux_gpu_training_cu102: false - enable_linux_gpu_training_cu111: false - enable_linux_rocm_training: true - enable_windows_cpu: false - enable_windows_gpu: false - enable_mac_cpu: false - enable_linux_arm: false +- stage: Python_Packaging + jobs: + - job: ROCm_build_environment + displayName: 'Construct ROCm wheels environment' + timeoutInMinutes: 300 + workspace: + clean: all + pool: AMD-GPU + steps: + - template: templates/ get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: >- + --build-arg TORCH_VERSION=1.11.0 + --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur + --build-arg BUILD_UID=$(id -u) + --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 + --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root + --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: + --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib + Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.3.1 + - template: templates/ get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: >- + --build-arg TORCH_VERSION=1.11.0 + --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur + --build-arg BUILD_UID=$(id -u) + --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 + --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root + --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: + --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib + Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm5.0.1 + + - job: ROCM_training_wheels + timeoutInMinutes: 180 + displayName: 'Build ROCm wheels (inside container)' + workspace: + clean: all + pool: AMD-GPU + dependsOn: + - ROCm_build_environment + strategy: + matrix: + Python37 Torch1100 Rocm431: + PythonVersion: '3.7' + TorchVersion: '1.10.0' + RocmVersion: '4.3.1' + Python38 Torch1100 Rocm431: + PythonVersion: '3.8' + TorchVersion: '1.10.0' + RocmVersion: '4.3.1' + Python39 Torch1100 Rocm431: + PythonVersion: '3.9' + TorchVersion: '1.10.0' + RocmVersion: '4.3.1' + Python37 Torch1100 Rocm501: + PythonVersion: '3.7' + TorchVersion: '1.10.0' + RocmVersion: '5.0.1' + Python38 Torch1100 Rocm501: + PythonVersion: '3.8' + TorchVersion: '1.10.0' + RocmVersion: '5.0.1' + Python39 Torch1100 Rocm501: + PythonVersion: '3.9' + TorchVersion: '1.10.0' + RocmVersion: '5.0.1' + steps: + + - checkout: self + clean: true + submodules: recursive + + - template: templates/ set-python-manylinux-variables-step.yml + + - task: CmdLine@2 + inputs: + script: | + docker run --rm \ + --privileged \ + --ipc=host \ + --network=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /onnxruntime_src \ + --entrypoint $(PythonManylinuxDir)/bin/python3 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + --user onnxruntimedev \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ + /onnxruntime_src/tools/ci_build/build.py \ + --config Release \ + --use_rocm \ + --rocm_version=$(RocmVersion) \ + --rocm_home=/opt/rocm \ + --nccl_home=/opt/rocm \ + --update \ + --parallel \ + --build_dir /build \ + --build \ + --build_wheel \ + --skip_tests \ + --enable_training + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Build onnxruntime (in container)' + + - script: |- + python3 orttraining/tools/ci_test/download_azure_blob_archive.py \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --target_dir $(Build.SourcesDirectory)/training_e2e_test_data \ + --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 + displayName: 'Download onnxruntime_training_data.zip data' + + - script: |- + echo "Tests will run using HIP_VISIBLES_DEVICES=$HIP_VISIBLE_DEVICES" + video_gid=$(getent group | awk '/video/ {split($0,a,":"); print(a[3])}') + echo "Found video_gid=$video_gid; attempting to set as pipeline variable" + echo "##vso[task.setvariable variable=video]$video_gid" + render_gid=$(getent group | awk '/render/ {split($0,a,":"); print(a[3])}') + echo "Found render_gid=$render_gid; attempting to set as pipeline variable" + echo "##vso[task.setvariable variable=render]$render_gid" + displayName: 'Find video and render gid to be mapped into container' + + - script: |- + echo "video=$video" + echo "render=$render" + docker run --rm \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --privileged \ + --ipc=host \ + --network=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /build/Release \ + --entrypoint /bin/bash \ + -e HIP_VISIBLE_DEVICES \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + --user onnxruntimedev \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ + /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh + displayName: 'Run onnxruntime unit tests (in container)' + + - script: |- + docker run --rm \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --privileged \ + --ipc=host \ + --network=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /onnxruntime_src \ + --entrypoint $(PythonManylinuxDir)/bin/python3 \ + -e HIP_VISIBLE_DEVICES \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + --user onnxruntimedev \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ + orttraining/tools/ci_test/run_batch_size_test.py \ + --binary_dir /build/Release \ + --model_root training_e2e_test_data/models \ + --gpu_sku MI100_32G + displayName: 'Run C++ BERT-L batch size test (in container)' + condition: succeededOrFailed() # ensure all tests are run + + - script: |- + docker run --rm \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --privileged \ + --ipc=host \ + --network=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /onnxruntime_src \ + --entrypoint $(PythonManylinuxDir)/bin/python3 \ + -e HIP_VISIBLE_DEVICES \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + --user onnxruntimedev \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ + orttraining/tools/ci_test/run_bert_perf_test.py \ + --binary_dir /build/Release \ + --model_root training_e2e_test_data/models \ + --training_data_root training_e2e_test_data/data \ + --gpu_sku MI100_32G + displayName: 'Run C++ BERT-L performance test (in container)' + condition: succeededOrFailed() # ensure all tests are run + + - script: |- + docker run --rm \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --privileged \ + --ipc=host \ + --network=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --workdir /onnxruntime_src \ + --entrypoint $(PythonManylinuxDir)/bin/python3 \ + -e HIP_VISIBLE_DEVICES \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + --user onnxruntimedev \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ + orttraining/tools/ci_test/run_convergence_test.py \ + --binary_dir /build/Release \ + --model_root training_e2e_test_data/models \ + --training_data_root training_e2e_test_data/data \ + --gpu_sku MI100_32G + displayName: 'Run C++ BERT-L convergence test (in container)' + condition: succeededOrFailed() # ensure all tests are run + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)' + Contents: 'Release/dist/*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: CmdLine@2 + displayName: 'Build Python Documentation' + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add $(video) \ + --group-add $(render) \ + --privileged \ + --ipc=host \ + --network=host \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --entrypoint /bin/bash \ + -e HIP_VISIBLE_DEVICES \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + -e PythonManylinuxDir=$(PythonManylinuxdir) \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ + /onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh + workingDirectory: $(Build.SourcesDirectory) + + - task: CopyFiles@2 + displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)' + condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 + inputs: + SourceFolder: '$(Build.BinariesDirectory)/docs/training/html' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/training_html_doc' + + - task: PublishBuildArtifacts@1 + displayName: 'Upload Rocm wheel as build artifact' + inputs: + ArtifactName: onnxruntime_rocm + + - script: | + python3 -m pip install azure-storage-blob==2.1.0 + files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ + echo ${files[0]} && \ + python3 tools/ci_build/upload_python_package_to_azure_storage_with_python.py \ + --python_wheel_path ${files[0]} \ + --account_name onnxruntimepackages \ + --account_key $(orttrainingpackagestorageaccountkey) \ + --container_name '$web' + condition: and(succeeded(), eq(variables['DRY_RUN'], '0')) + displayName: 'Upload Rocm wheel to release repository' + + - template: templates/ component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - template: templates/ clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index f0ea943195..38363a49d2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -25,11 +25,6 @@ parameters: type: boolean default: false -- name: enable_linux_rocm_training - displayName: 'Whether Linux ROCM package is built.' - type: boolean - default: false - - name: enable_windows_cpu displayName: 'Whether Windows CPU package is built.' type: boolean @@ -284,332 +279,6 @@ stages: - template: clean-agent-build-directory-step.yml - - ${{ if eq(parameters.enable_linux_rocm_training, true) }}: - - job: ROCm_build_environment - displayName: 'Construct ROCm wheels environment' - timeoutInMinutes: 300 - workspace: - clean: all - pool: AMD-GPU - steps: - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: >- - --build-arg TORCH_VERSION=1.10.0 - --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur - --build-arg BUILD_UID=$(id -u) - --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 - --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root - --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: - --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib - Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.2 - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: >- - --build-arg TORCH_VERSION=1.10.0 - --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur - --build-arg BUILD_UID=$(id -u) - --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 - --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root - --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: - --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib - Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.3.1 - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: >- - --build-arg TORCH_VERSION=1.10.0 - --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur - --build-arg BUILD_UID=$(id -u) - --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 - --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root - --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: - --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib - Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm5.0.1 - - - job: ROCM_training_wheels - timeoutInMinutes: 180 - displayName: 'Build ROCm wheels (inside container)' - workspace: - clean: all - pool: AMD-GPU - dependsOn: - - ROCm_build_environment - strategy: - matrix: - Python37 Torch1100 Rocm42: - PythonVersion: '3.7' - TorchVersion: '1.10.0' - RocmVersion: '4.2' - Python38 Torch1100 Rocm42: - PythonVersion: '3.8' - TorchVersion: '1.10.0' - RocmVersion: '4.2' - Python39 Torch1100 Rocm42: - PythonVersion: '3.9' - TorchVersion: '1.10.0' - RocmVersion: '4.2' - Python37 Torch1100 Rocm431: - PythonVersion: '3.7' - TorchVersion: '1.10.0' - RocmVersion: '4.3.1' - Python38 Torch1100 Rocm431: - PythonVersion: '3.8' - TorchVersion: '1.10.0' - RocmVersion: '4.3.1' - Python39 Torch1100 Rocm431: - PythonVersion: '3.9' - TorchVersion: '1.10.0' - RocmVersion: '4.3.1' - Python37 Torch1100 Rocm501: - PythonVersion: '3.7' - TorchVersion: '1.10.0' - RocmVersion: '5.0.1' - Python38 Torch1100 Rocm501: - PythonVersion: '3.8' - TorchVersion: '1.10.0' - RocmVersion: '5.0.1' - Python39 Torch1100 Rocm501: - PythonVersion: '3.9' - TorchVersion: '1.10.0' - RocmVersion: '5.0.1' - steps: - - - checkout: self - clean: true - submodules: recursive - - - template: set-python-manylinux-variables-step.yml - - - task: CmdLine@2 - inputs: - script: | - docker run --rm \ - --privileged \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /onnxruntime_src \ - --entrypoint $(PythonManylinuxDir)/bin/python3 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - --user onnxruntimedev \ - onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ - /onnxruntime_src/tools/ci_build/build.py \ - --config Release \ - --use_rocm \ - --rocm_version=$(RocmVersion) \ - --rocm_home=/opt/rocm \ - --nccl_home=/opt/rocm \ - --update \ - --parallel \ - --build_dir /build \ - --build \ - --build_wheel \ - --skip_tests \ - ${{ parameters.build_py_parameters }} - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Build onnxruntime (in container)' - - - script: |- - python3 orttraining/tools/ci_test/download_azure_blob_archive.py \ - --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ - --target_dir $(Build.SourcesDirectory)/training_e2e_test_data \ - --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 - displayName: 'Download onnxruntime_training_data.zip data' - - - script: |- - echo "Tests will run using HIP_VISIBLES_DEVICES=$HIP_VISIBLE_DEVICES" - video_gid=$(getent group | awk '/video/ {split($0,a,":"); print(a[3])}') - echo "Found video_gid=$video_gid; attempting to set as pipeline variable" - echo "##vso[task.setvariable variable=video]$video_gid" - render_gid=$(getent group | awk '/render/ {split($0,a,":"); print(a[3])}') - echo "Found render_gid=$render_gid; attempting to set as pipeline variable" - echo "##vso[task.setvariable variable=render]$render_gid" - displayName: 'Find video and render gid to be mapped into container' - - - script: |- - echo "video=$video" - echo "render=$render" - docker run --rm \ - --device=/dev/kfd \ - --device=/dev/dri \ - --group-add $(video) \ - --group-add $(render) \ - --privileged \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /build/Release \ - --entrypoint /bin/bash \ - -e HIP_VISIBLE_DEVICES \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - --user onnxruntimedev \ - onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ - /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh - displayName: 'Run onnxruntime unit tests (in container)' - - - script: |- - docker run --rm \ - --device=/dev/kfd \ - --device=/dev/dri \ - --group-add $(video) \ - --group-add $(render) \ - --privileged \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /onnxruntime_src \ - --entrypoint $(PythonManylinuxDir)/bin/python3 \ - -e HIP_VISIBLE_DEVICES \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - --user onnxruntimedev \ - onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ - orttraining/tools/ci_test/run_batch_size_test.py \ - --binary_dir /build/Release \ - --model_root training_e2e_test_data/models \ - --gpu_sku MI100_32G - displayName: 'Run C++ BERT-L batch size test (in container)' - condition: succeededOrFailed() # ensure all tests are run - - - script: |- - docker run --rm \ - --device=/dev/kfd \ - --device=/dev/dri \ - --group-add $(video) \ - --group-add $(render) \ - --privileged \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /onnxruntime_src \ - --entrypoint $(PythonManylinuxDir)/bin/python3 \ - -e HIP_VISIBLE_DEVICES \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - --user onnxruntimedev \ - onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ - orttraining/tools/ci_test/run_bert_perf_test.py \ - --binary_dir /build/Release \ - --model_root training_e2e_test_data/models \ - --training_data_root training_e2e_test_data/data \ - --gpu_sku MI100_32G - displayName: 'Run C++ BERT-L performance test (in container)' - condition: succeededOrFailed() # ensure all tests are run - - - script: |- - docker run --rm \ - --device=/dev/kfd \ - --device=/dev/dri \ - --group-add $(video) \ - --group-add $(render) \ - --privileged \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /onnxruntime_src \ - --entrypoint $(PythonManylinuxDir)/bin/python3 \ - -e HIP_VISIBLE_DEVICES \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - --user onnxruntimedev \ - onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ - orttraining/tools/ci_test/run_convergence_test.py \ - --binary_dir /build/Release \ - --model_root training_e2e_test_data/models \ - --training_data_root training_e2e_test_data/data \ - --gpu_sku MI100_32G - displayName: 'Run C++ BERT-L convergence test (in container)' - condition: succeededOrFailed() # ensure all tests are run - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)' - Contents: 'Release/dist/*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: CmdLine@2 - displayName: 'Build Python Documentation' - condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm \ - --device=/dev/kfd \ - --device=/dev/dri \ - --group-add $(video) \ - --group-add $(render) \ - --privileged \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --entrypoint /bin/bash \ - -e HIP_VISIBLE_DEVICES \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - -e PythonManylinuxDir=$(PythonManylinuxdir) \ - onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \ - /onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh - workingDirectory: $(Build.SourcesDirectory) - - - task: CopyFiles@2 - displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)' - condition: and(succeeded(), ne(variables['PythonVersion'], '3.9')) # tensorflow not available on python 3.9 - inputs: - SourceFolder: '$(Build.BinariesDirectory)/docs/training/html' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/training_html_doc' - - - task: PublishBuildArtifacts@1 - displayName: 'Upload Rocm wheel as build artifact' - inputs: - ArtifactName: onnxruntime_rocm - - - script: | - python3 -m pip install azure-storage-blob==2.1.0 - files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ - echo ${files[0]} && \ - python3 tools/ci_build/upload_python_package_to_azure_storage.py \ - --python_wheel_path ${files[0]} \ - --account_name onnxruntimepackages \ - --account_key $(orttrainingpackagestorageaccountkey) \ - --container_name '$web' - condition: and(succeeded(), eq(variables['DRY_RUN'], '0')) - displayName: 'Upload Rocm wheel to release repository' - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - template: clean-agent-build-directory-step.yml - - ${{ if eq(parameters.enable_windows_cpu, true) }}: - job: Windows_py_Wheels pool: 'Win-CPU-2021' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml index 6cc5fc6305..85dd578255 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml @@ -206,22 +206,16 @@ stages: inputs: ArtifactName: onnxruntime_gpu - - task: AzureCLI@2 + - task: CmdLine@2 + displayName: 'Upload wheel' condition: and(succeeded(), eq(variables['UploadWheel'], 'yes')) inputs: - azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - python3 -m pip install azure-storage-blob==2.1.0 + script: | + set -e -x files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \ echo ${files[0]} && \ python3 tools/ci_build/upload_python_package_to_azure_storage.py \ - --python_wheel_path ${files[0]} \ - --account_name onnxruntimepackages \ - --account_key $(orttrainingpackagestorageaccountkey) \ - --container_name '$web' - displayName: + --python_wheel_path ${files[0]} - template: component-governance-component-detection-steps.yml parameters: diff --git a/tools/ci_build/upload_python_package_to_azure_storage.py b/tools/ci_build/upload_python_package_to_azure_storage.py index da7e4a8ea5..a0b3ad5836 100755 --- a/tools/ci_build/upload_python_package_to_azure_storage.py +++ b/tools/ci_build/upload_python_package_to_azure_storage.py @@ -5,7 +5,10 @@ import os import argparse import warnings -from azure.storage.blob import BlockBlobService, ContentSettings +import subprocess +import logging + +log = logging.getLogger("Build") def parse_nightly_and_local_version_from_whl_name(blob_name): @@ -21,14 +24,14 @@ def parse_nightly_and_local_version_from_whl_name(blob_name): return night_build, blob_name[start:end] -def upload_whl(python_wheel_path, account_name, account_key, container_name): - block_blob_service = BlockBlobService( - account_name=account_name, - account_key=account_key - ) +def run_subprocess(args, cwd=None): + log.debug("Running subprocess in '{0}'\n{1}".format(cwd or os.getcwd(), args)) + return subprocess.run(args, cwd=cwd, check=True) + +def upload_whl(python_wheel_path): blob_name = os.path.basename(python_wheel_path) - block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path) + run_subprocess(['azcopy', 'cp', python_wheel_path, 'https://onnxruntimepackages.blob.core.windows.net/$web/']) nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name) if local_version: @@ -38,7 +41,8 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name): download_path_to_html = "./onnxruntime_{}.html".format(nightly_build) - block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html) + run_subprocess(['azcopy', 'cp', 'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name, + download_path_to_html]) blob_name_plus_replaced = blob_name.replace('+', '%2B') with open(download_path_to_html) as f: @@ -54,25 +58,16 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name): f.write("%s\n" % item) else: warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html)) - - content_settings = ContentSettings(content_type='text/html') - block_blob_service.create_blob_from_path( - container_name, - html_blob_name, - download_path_to_html, - content_settings=content_settings) + run_subprocess(['azcopy', 'cp', download_path_to_html, + 'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name, + '--content-type', 'text/html', '--overwrite', 'true']) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Upload python whl to azure storage.") parser.add_argument("--python_wheel_path", type=str, help="path to python wheel") - parser.add_argument("--account_name", type=str, help="account name") - parser.add_argument("--account_key", type=str, help="account key") - parser.add_argument("--container_name", type=str, help="container name") - # TODO: figure out a way to secure args.account_key to prevent later code changes - # that may accidentally print out it to the console. args = parser.parse_args() - upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name) + upload_whl(args.python_wheel_path) diff --git a/tools/ci_build/upload_python_package_to_azure_storage_with_python.py b/tools/ci_build/upload_python_package_to_azure_storage_with_python.py new file mode 100644 index 0000000000..da7e4a8ea5 --- /dev/null +++ b/tools/ci_build/upload_python_package_to_azure_storage_with_python.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import argparse +import warnings +from azure.storage.blob import BlockBlobService, ContentSettings + + +def parse_nightly_and_local_version_from_whl_name(blob_name): + night_build = 'nightly' if blob_name.find(".dev") > 0 else 'stable' + + start = blob_name.find("+") + if start == -1: + return night_build, None + start = start + 1 + end = blob_name.find("-", start) + if end == -1: + return night_build, None + return night_build, blob_name[start:end] + + +def upload_whl(python_wheel_path, account_name, account_key, container_name): + block_blob_service = BlockBlobService( + account_name=account_name, + account_key=account_key + ) + + blob_name = os.path.basename(python_wheel_path) + block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path) + + nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name) + if local_version: + html_blob_name = 'onnxruntime_{}_{}.html'.format(nightly_build, local_version) + else: + html_blob_name = 'onnxruntime_{}.html'.format(nightly_build) + + download_path_to_html = "./onnxruntime_{}.html".format(nightly_build) + + block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html) + + blob_name_plus_replaced = blob_name.replace('+', '%2B') + with open(download_path_to_html) as f: + lines = f.read().splitlines() + + new_line = '{blobname}
'.format(blobname=blob_name_plus_replaced) + if new_line not in lines: + lines.append(new_line) + lines.sort() + + with open(download_path_to_html, 'w') as f: + for item in lines: + f.write("%s\n" % item) + else: + warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html)) + + content_settings = ContentSettings(content_type='text/html') + block_blob_service.create_blob_from_path( + container_name, + html_blob_name, + download_path_to_html, + content_settings=content_settings) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Upload python whl to azure storage.") + + parser.add_argument("--python_wheel_path", type=str, help="path to python wheel") + parser.add_argument("--account_name", type=str, help="account name") + parser.add_argument("--account_key", type=str, help="account key") + parser.add_argument("--container_name", type=str, help="container name") + + # TODO: figure out a way to secure args.account_key to prevent later code changes + # that may accidentally print out it to the console. + args = parser.parse_args() + + upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name)