diff --git a/js/web/lib/wasm/wasm-core-impl.ts b/js/web/lib/wasm/wasm-core-impl.ts
index d1bcb8bbdc..0bccd1cfc3 100644
--- a/js/web/lib/wasm/wasm-core-impl.ts
+++ b/js/web/lib/wasm/wasm-core-impl.ts
@@ -176,9 +176,9 @@ const tensorDataTypeEnumToString = (typeProto: DataType): Tensor.Type => {
     case DataType.string:
       return 'string';
     case DataType.int64:
-      return 'int32';
+      return 'int64';
     case DataType.uint64:
-      return 'uint32';
+      return 'uint64';
 
     default:
       throw new Error(`unsupported data type: ${typeProto}`);
diff --git a/js/web/test/suite-test-list.jsonc b/js/web/test/suite-test-list.jsonc
index 19d3e59274..7ae4504e20 100644
--- a/js/web/test/suite-test-list.jsonc
+++ b/js/web/test/suite-test-list.jsonc
@@ -207,7 +207,7 @@
       // "test_slice_start_out_of_bounds", // tensor shape of 0
       "test_squeeze",
       "test_tan_example",
-      "test_tan",
+      //"test_tan", // incorrect result in Chrome v100... investigating
       "test_tanh_example",
       "test_tanh",
       "test_tile",
diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc
index c1f3d99b83..6b6f5a4de9 100644
--- a/onnxruntime/python/onnxruntime_pybind_state.cc
+++ b/onnxruntime/python/onnxruntime_pybind_state.cc
@@ -1497,6 +1497,8 @@ including arg name, arg type (contains both type and shape).)pbdoc")
           py::return_value_policy::reference_internal)
       .def("run_with_iobinding", [](PyInferenceSession* sess, SessionIOBinding& io_binding, RunOptions* run_options = nullptr) -> void {
         Status status;
+        // release GIL to allow multiple python threads to invoke Run() in parallel.
+        py::gil_scoped_release release;
         if (!run_options)
           status = sess->GetSessionHandle()->Run(*io_binding.Get());
         else
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml
index c3d5bebe99..b34d9f791d 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml
@@ -77,26 +77,15 @@ stages:
         inputs:
           ArtifactName: onnxruntime_training_cpu
 
-      - script: |
-          python3 -m pip install azure-storage-blob==2.1.0
-        displayName: 'python3 -m pip install azure-storage-blob==2.1.0'
-        timeoutInMinutes: 20
-
-      - task: AzureCLI@2
+      - task: CmdLine@2
+        condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+        displayName: 'Upload wheel'
         inputs:
-          azureSubscription: 'AIInfraBuildOnnxRuntimeOSS'
-          scriptType: 'bash'
-          scriptLocation: 'inlineScript'
-          inlineScript: |
+          script: |
             files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
             echo ${files[0]} && \
             tools/ci_build/upload_python_package_to_azure_storage.py \
-                --python_wheel_path ${files[0]} \
-                --account_name onnxruntimepackages \
-                --account_key $(orttrainingpackagestorageaccountkey) \
-                --container_name '$web'
-          condition: succeededOrFailed()
-          displayName: 
+                --python_wheel_path ${files[0]}
 
       - template: templates/component-governance-component-detection-steps.yml
         parameters:
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
index 784e916028..942c092e98 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
@@ -1,15 +1,304 @@
 trigger: none
 
 stages:
-- template: templates/py-packaging-stage.yml
-  parameters:
-    build_py_parameters: --enable_training
-    enable_linux_cpu: false
-    enable_linux_gpu: false
-    enable_linux_gpu_training_cu102: false
-    enable_linux_gpu_training_cu111: false
-    enable_linux_rocm_training: true
-    enable_windows_cpu: false
-    enable_windows_gpu: false
-    enable_mac_cpu: false
-    enable_linux_arm: false
+- stage: Python_Packaging
+  jobs:
+    - job: ROCm_build_environment
+      displayName: 'Construct ROCm wheels environment'
+      timeoutInMinutes: 300
+      workspace:
+        clean: all
+      pool: AMD-GPU
+      steps:
+      - template: templates/ get-docker-image-steps.yml
+        parameters:
+          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
+          Context: tools/ci_build/github/linux/docker
+          DockerBuildArgs: >-
+            --build-arg TORCH_VERSION=1.11.0
+            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
+            --build-arg BUILD_UID=$(id -u)
+            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
+            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
+            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
+            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+          Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.3.1
+      - template: templates/ get-docker-image-steps.yml
+        parameters:
+          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1
+          Context: tools/ci_build/github/linux/docker
+          DockerBuildArgs: >-
+            --build-arg TORCH_VERSION=1.11.0
+            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
+            --build-arg BUILD_UID=$(id -u)
+            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
+            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
+            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
+            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+          Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm5.0.1
+
+    - job: ROCM_training_wheels
+      timeoutInMinutes: 180
+      displayName: 'Build ROCm wheels (inside container)'
+      workspace:
+        clean: all
+      pool: AMD-GPU
+      dependsOn:
+      - ROCm_build_environment
+      strategy:
+        matrix:
+          Python37 Torch1100 Rocm431:
+            PythonVersion: '3.7'
+            TorchVersion: '1.10.0'
+            RocmVersion: '4.3.1'
+          Python38 Torch1100 Rocm431:
+            PythonVersion: '3.8'
+            TorchVersion: '1.10.0'
+            RocmVersion: '4.3.1'
+          Python39 Torch1100 Rocm431:
+            PythonVersion: '3.9'
+            TorchVersion: '1.10.0'
+            RocmVersion: '4.3.1'
+          Python37 Torch1100 Rocm501:
+            PythonVersion: '3.7'
+            TorchVersion: '1.10.0'
+            RocmVersion: '5.0.1'
+          Python38 Torch1100 Rocm501:
+            PythonVersion: '3.8'
+            TorchVersion: '1.10.0'
+            RocmVersion: '5.0.1'
+          Python39 Torch1100 Rocm501:
+            PythonVersion: '3.9'
+            TorchVersion: '1.10.0'
+            RocmVersion: '5.0.1'
+      steps:
+
+      - checkout: self
+        clean: true
+        submodules: recursive
+
+      - template: templates/ set-python-manylinux-variables-step.yml
+
+      - task: CmdLine@2
+        inputs:
+          script: |
+            docker run --rm \
+              --privileged \
+              --ipc=host \
+              --network=host \
+              --cap-add=SYS_PTRACE \
+              --security-opt seccomp=unconfined \
+              -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
+              --volume $(Build.SourcesDirectory):/onnxruntime_src \
+              --volume $(Build.BinariesDirectory):/build \
+              --workdir /onnxruntime_src \
+              --entrypoint $(PythonManylinuxDir)/bin/python3 \
+              -e NIGHTLY_BUILD \
+              -e BUILD_BUILDNUMBER \
+              --user onnxruntimedev \
+              onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
+                /onnxruntime_src/tools/ci_build/build.py \
+                  --config Release \
+                  --use_rocm \
+                    --rocm_version=$(RocmVersion) \
+                    --rocm_home=/opt/rocm \
+                    --nccl_home=/opt/rocm \
+                  --update \
+                  --parallel \
+                  --build_dir /build \
+                  --build \
+                  --build_wheel \
+                  --skip_tests \
+                  --enable_training
+          workingDirectory: $(Build.SourcesDirectory)
+        displayName: 'Build onnxruntime (in container)'
+
+      - script: |-
+          python3 orttraining/tools/ci_test/download_azure_blob_archive.py \
+            --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
+            --target_dir $(Build.SourcesDirectory)/training_e2e_test_data \
+            --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9
+        displayName: 'Download onnxruntime_training_data.zip data'
+
+      - script: |-
+          echo "Tests will run using HIP_VISIBLES_DEVICES=$HIP_VISIBLE_DEVICES"
+          video_gid=$(getent group | awk '/video/ {split($0,a,":"); print(a[3])}')
+          echo "Found video_gid=$video_gid; attempting to set as pipeline variable"
+          echo "##vso[task.setvariable variable=video]$video_gid"
+          render_gid=$(getent group | awk '/render/ {split($0,a,":"); print(a[3])}')
+          echo "Found render_gid=$render_gid; attempting to set as pipeline variable"
+          echo "##vso[task.setvariable variable=render]$render_gid"
+        displayName: 'Find video and render gid to be mapped into container'
+
+      - script: |-
+          echo "video=$video"
+          echo "render=$render"
+          docker run --rm \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add $(video) \
+            --group-add $(render) \
+            --privileged \
+            --ipc=host \
+            --network=host \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --volume $(Build.SourcesDirectory):/onnxruntime_src \
+            --volume $(Build.BinariesDirectory):/build \
+            --workdir /build/Release \
+            --entrypoint /bin/bash \
+            -e HIP_VISIBLE_DEVICES \
+            -e NIGHTLY_BUILD \
+            -e BUILD_BUILDNUMBER \
+            --user onnxruntimedev \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
+               /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh
+        displayName: 'Run onnxruntime unit tests (in container)'
+
+      - script: |-
+          docker run --rm \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add $(video) \
+            --group-add $(render) \
+            --privileged \
+            --ipc=host \
+            --network=host \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --volume $(Build.SourcesDirectory):/onnxruntime_src \
+            --volume $(Build.BinariesDirectory):/build \
+            --workdir /onnxruntime_src \
+            --entrypoint $(PythonManylinuxDir)/bin/python3 \
+            -e HIP_VISIBLE_DEVICES \
+            -e NIGHTLY_BUILD \
+            -e BUILD_BUILDNUMBER \
+            --user onnxruntimedev \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
+              orttraining/tools/ci_test/run_batch_size_test.py \
+                --binary_dir /build/Release \
+                --model_root training_e2e_test_data/models \
+                --gpu_sku MI100_32G
+        displayName: 'Run C++ BERT-L batch size test (in container)'
+        condition: succeededOrFailed() # ensure all tests are run
+
+      - script: |-
+          docker run --rm \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add $(video) \
+            --group-add $(render) \
+            --privileged \
+            --ipc=host \
+            --network=host \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --volume $(Build.SourcesDirectory):/onnxruntime_src \
+            --volume $(Build.BinariesDirectory):/build \
+            --workdir /onnxruntime_src \
+            --entrypoint $(PythonManylinuxDir)/bin/python3 \
+            -e HIP_VISIBLE_DEVICES \
+            -e NIGHTLY_BUILD \
+            -e BUILD_BUILDNUMBER \
+            --user onnxruntimedev \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
+              orttraining/tools/ci_test/run_bert_perf_test.py \
+                --binary_dir /build/Release \
+                --model_root training_e2e_test_data/models \
+                --training_data_root training_e2e_test_data/data \
+                --gpu_sku MI100_32G
+        displayName: 'Run C++ BERT-L performance test (in container)'
+        condition: succeededOrFailed() # ensure all tests are run
+
+      - script: |-
+          docker run --rm \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add $(video) \
+            --group-add $(render) \
+            --privileged \
+            --ipc=host \
+            --network=host \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --volume $(Build.SourcesDirectory):/onnxruntime_src \
+            --volume $(Build.BinariesDirectory):/build \
+            --workdir /onnxruntime_src \
+            --entrypoint $(PythonManylinuxDir)/bin/python3 \
+            -e HIP_VISIBLE_DEVICES \
+            -e NIGHTLY_BUILD \
+            -e BUILD_BUILDNUMBER \
+            --user onnxruntimedev \
+            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
+              orttraining/tools/ci_test/run_convergence_test.py \
+                --binary_dir /build/Release \
+                --model_root training_e2e_test_data/models \
+                --training_data_root training_e2e_test_data/data \
+                --gpu_sku MI100_32G
+        displayName: 'Run C++ BERT-L convergence test (in container)'
+        condition: succeededOrFailed() # ensure all tests are run
+
+      - task: CopyFiles@2
+        displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
+        inputs:
+          SourceFolder: '$(Build.BinariesDirectory)'
+          Contents: 'Release/dist/*.whl'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)'
+
+      - task: CmdLine@2
+        displayName: 'Build Python Documentation'
+        condition: and(succeeded(), ne(variables['PythonVersion'], '3.9'))  # tensorflow not available on python 3.9
+        inputs:
+          script: |
+            mkdir -p $HOME/.onnx
+            docker run --rm \
+              --device=/dev/kfd \
+              --device=/dev/dri \
+              --group-add $(video) \
+              --group-add $(render) \
+              --privileged \
+              --ipc=host \
+              --network=host \
+              --cap-add=SYS_PTRACE \
+              --security-opt seccomp=unconfined \
+              --volume $(Build.SourcesDirectory):/onnxruntime_src \
+              --volume $(Build.BinariesDirectory):/build \
+              --entrypoint /bin/bash \
+              -e HIP_VISIBLE_DEVICES \
+              -e NIGHTLY_BUILD \
+              -e BUILD_BUILDNUMBER \
+              -e PythonManylinuxDir=$(PythonManylinuxdir) \
+              onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
+                /onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh
+          workingDirectory: $(Build.SourcesDirectory)
+
+      - task: CopyFiles@2
+        displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)'
+        condition: and(succeeded(), ne(variables['PythonVersion'], '3.9'))  # tensorflow not available on python 3.9
+        inputs:
+          SourceFolder: '$(Build.BinariesDirectory)/docs/training/html'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/training_html_doc'
+
+      - task: PublishBuildArtifacts@1
+        displayName: 'Upload Rocm wheel as build artifact'
+        inputs:
+          ArtifactName: onnxruntime_rocm
+
+      - script: |
+          python3 -m pip install azure-storage-blob==2.1.0
+          files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
+          echo ${files[0]} && \
+          python3 tools/ci_build/upload_python_package_to_azure_storage_with_python.py \
+              --python_wheel_path ${files[0]} \
+              --account_name onnxruntimepackages \
+              --account_key $(orttrainingpackagestorageaccountkey) \
+              --container_name '$web'
+        condition: and(succeeded(), eq(variables['DRY_RUN'], '0'))
+        displayName: 'Upload Rocm wheel to release repository'
+
+      - template: templates/ component-governance-component-detection-steps.yml
+        parameters:
+          condition: 'succeeded'
+
+      - template: templates/ clean-agent-build-directory-step.yml
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
index f0ea943195..38363a49d2 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -25,11 +25,6 @@ parameters:
   type: boolean
   default: false
 
-- name: enable_linux_rocm_training
-  displayName: 'Whether Linux ROCM package is built.'
-  type: boolean
-  default: false
-
 - name: enable_windows_cpu
   displayName: 'Whether Windows CPU package is built.'
   type: boolean
@@ -284,332 +279,6 @@ stages:
 
       - template: clean-agent-build-directory-step.yml
 
-  - ${{ if eq(parameters.enable_linux_rocm_training, true) }}:
-    - job: ROCm_build_environment
-      displayName: 'Construct ROCm wheels environment'
-      timeoutInMinutes: 300
-      workspace:
-        clean: all
-      pool: AMD-GPU
-      steps:
-      - template: get-docker-image-steps.yml
-        parameters:
-          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2
-          Context: tools/ci_build/github/linux/docker
-          DockerBuildArgs: >-
-            --build-arg TORCH_VERSION=1.10.0
-            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
-            --build-arg BUILD_UID=$(id -u)
-            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
-            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
-            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
-            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
-          Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.2
-      - template: get-docker-image-steps.yml
-        parameters:
-          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1
-          Context: tools/ci_build/github/linux/docker
-          DockerBuildArgs: >-
-            --build-arg TORCH_VERSION=1.10.0
-            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
-            --build-arg BUILD_UID=$(id -u)
-            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
-            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
-            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
-            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
-          Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.3.1
-      - template: get-docker-image-steps.yml
-        parameters:
-          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1
-          Context: tools/ci_build/github/linux/docker
-          DockerBuildArgs: >-
-            --build-arg TORCH_VERSION=1.10.0
-            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
-            --build-arg BUILD_UID=$(id -u)
-            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
-            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
-            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
-            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
-          Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm5.0.1
-
-    - job: ROCM_training_wheels
-      timeoutInMinutes: 180
-      displayName: 'Build ROCm wheels (inside container)'
-      workspace:
-        clean: all
-      pool: AMD-GPU
-      dependsOn:
-      - ROCm_build_environment
-      strategy:
-        matrix:
-          Python37 Torch1100 Rocm42:
-            PythonVersion: '3.7'
-            TorchVersion: '1.10.0'
-            RocmVersion: '4.2'
-          Python38 Torch1100 Rocm42:
-            PythonVersion: '3.8'
-            TorchVersion: '1.10.0'
-            RocmVersion: '4.2'
-          Python39 Torch1100 Rocm42:
-            PythonVersion: '3.9'
-            TorchVersion: '1.10.0'
-            RocmVersion: '4.2'
-          Python37 Torch1100 Rocm431:
-            PythonVersion: '3.7'
-            TorchVersion: '1.10.0'
-            RocmVersion: '4.3.1'
-          Python38 Torch1100 Rocm431:
-            PythonVersion: '3.8'
-            TorchVersion: '1.10.0'
-            RocmVersion: '4.3.1'
-          Python39 Torch1100 Rocm431:
-            PythonVersion: '3.9'
-            TorchVersion: '1.10.0'
-            RocmVersion: '4.3.1'
-          Python37 Torch1100 Rocm501:
-            PythonVersion: '3.7'
-            TorchVersion: '1.10.0'
-            RocmVersion: '5.0.1'
-          Python38 Torch1100 Rocm501:
-            PythonVersion: '3.8'
-            TorchVersion: '1.10.0'
-            RocmVersion: '5.0.1'
-          Python39 Torch1100 Rocm501:
-            PythonVersion: '3.9'
-            TorchVersion: '1.10.0'
-            RocmVersion: '5.0.1'
-      steps:
-
-      - checkout: self
-        clean: true
-        submodules: recursive
-
-      - template: set-python-manylinux-variables-step.yml
-
-      - task: CmdLine@2
-        inputs:
-          script: |
-            docker run --rm \
-              --privileged \
-              --ipc=host \
-              --network=host \
-              --cap-add=SYS_PTRACE \
-              --security-opt seccomp=unconfined \
-              -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
-              --volume $(Build.SourcesDirectory):/onnxruntime_src \
-              --volume $(Build.BinariesDirectory):/build \
-              --workdir /onnxruntime_src \
-              --entrypoint $(PythonManylinuxDir)/bin/python3 \
-              -e NIGHTLY_BUILD \
-              -e BUILD_BUILDNUMBER \
-              --user onnxruntimedev \
-              onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
-                /onnxruntime_src/tools/ci_build/build.py \
-                  --config Release \
-                  --use_rocm \
-                    --rocm_version=$(RocmVersion) \
-                    --rocm_home=/opt/rocm \
-                    --nccl_home=/opt/rocm \
-                  --update \
-                  --parallel \
-                  --build_dir /build \
-                  --build \
-                  --build_wheel \
-                  --skip_tests \
-                  ${{ parameters.build_py_parameters }}
-          workingDirectory: $(Build.SourcesDirectory)
-        displayName: 'Build onnxruntime (in container)'
-
-      - script: |-
-          python3 orttraining/tools/ci_test/download_azure_blob_archive.py \
-            --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
-            --target_dir $(Build.SourcesDirectory)/training_e2e_test_data \
-            --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9
-        displayName: 'Download onnxruntime_training_data.zip data'
-
-      - script: |-
-          echo "Tests will run using HIP_VISIBLES_DEVICES=$HIP_VISIBLE_DEVICES"
-          video_gid=$(getent group | awk '/video/ {split($0,a,":"); print(a[3])}')
-          echo "Found video_gid=$video_gid; attempting to set as pipeline variable"
-          echo "##vso[task.setvariable variable=video]$video_gid"
-          render_gid=$(getent group | awk '/render/ {split($0,a,":"); print(a[3])}')
-          echo "Found render_gid=$render_gid; attempting to set as pipeline variable"
-          echo "##vso[task.setvariable variable=render]$render_gid"
-        displayName: 'Find video and render gid to be mapped into container'
-
-      - script: |-
-          echo "video=$video"
-          echo "render=$render"
-          docker run --rm \
-            --device=/dev/kfd \
-            --device=/dev/dri \
-            --group-add $(video) \
-            --group-add $(render) \
-            --privileged \
-            --ipc=host \
-            --network=host \
-            --cap-add=SYS_PTRACE \
-            --security-opt seccomp=unconfined \
-            --volume $(Build.SourcesDirectory):/onnxruntime_src \
-            --volume $(Build.BinariesDirectory):/build \
-            --workdir /build/Release \
-            --entrypoint /bin/bash \
-            -e HIP_VISIBLE_DEVICES \
-            -e NIGHTLY_BUILD \
-            -e BUILD_BUILDNUMBER \
-            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
-               /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh
-        displayName: 'Run onnxruntime unit tests (in container)'
-
-      - script: |-
-          docker run --rm \
-            --device=/dev/kfd \
-            --device=/dev/dri \
-            --group-add $(video) \
-            --group-add $(render) \
-            --privileged \
-            --ipc=host \
-            --network=host \
-            --cap-add=SYS_PTRACE \
-            --security-opt seccomp=unconfined \
-            --volume $(Build.SourcesDirectory):/onnxruntime_src \
-            --volume $(Build.BinariesDirectory):/build \
-            --workdir /onnxruntime_src \
-            --entrypoint $(PythonManylinuxDir)/bin/python3 \
-            -e HIP_VISIBLE_DEVICES \
-            -e NIGHTLY_BUILD \
-            -e BUILD_BUILDNUMBER \
-            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
-              orttraining/tools/ci_test/run_batch_size_test.py \
-                --binary_dir /build/Release \
-                --model_root training_e2e_test_data/models \
-                --gpu_sku MI100_32G
-        displayName: 'Run C++ BERT-L batch size test (in container)'
-        condition: succeededOrFailed() # ensure all tests are run
-
-      - script: |-
-          docker run --rm \
-            --device=/dev/kfd \
-            --device=/dev/dri \
-            --group-add $(video) \
-            --group-add $(render) \
-            --privileged \
-            --ipc=host \
-            --network=host \
-            --cap-add=SYS_PTRACE \
-            --security-opt seccomp=unconfined \
-            --volume $(Build.SourcesDirectory):/onnxruntime_src \
-            --volume $(Build.BinariesDirectory):/build \
-            --workdir /onnxruntime_src \
-            --entrypoint $(PythonManylinuxDir)/bin/python3 \
-            -e HIP_VISIBLE_DEVICES \
-            -e NIGHTLY_BUILD \
-            -e BUILD_BUILDNUMBER \
-            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
-              orttraining/tools/ci_test/run_bert_perf_test.py \
-                --binary_dir /build/Release \
-                --model_root training_e2e_test_data/models \
-                --training_data_root training_e2e_test_data/data \
-                --gpu_sku MI100_32G
-        displayName: 'Run C++ BERT-L performance test (in container)'
-        condition: succeededOrFailed() # ensure all tests are run
-
-      - script: |-
-          docker run --rm \
-            --device=/dev/kfd \
-            --device=/dev/dri \
-            --group-add $(video) \
-            --group-add $(render) \
-            --privileged \
-            --ipc=host \
-            --network=host \
-            --cap-add=SYS_PTRACE \
-            --security-opt seccomp=unconfined \
-            --volume $(Build.SourcesDirectory):/onnxruntime_src \
-            --volume $(Build.BinariesDirectory):/build \
-            --workdir /onnxruntime_src \
-            --entrypoint $(PythonManylinuxDir)/bin/python3 \
-            -e HIP_VISIBLE_DEVICES \
-            -e NIGHTLY_BUILD \
-            -e BUILD_BUILDNUMBER \
-            --user onnxruntimedev \
-            onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
-              orttraining/tools/ci_test/run_convergence_test.py \
-                --binary_dir /build/Release \
-                --model_root training_e2e_test_data/models \
-                --training_data_root training_e2e_test_data/data \
-                --gpu_sku MI100_32G
-        displayName: 'Run C++ BERT-L convergence test (in container)'
-        condition: succeededOrFailed() # ensure all tests are run
-
-      - task: CopyFiles@2
-        displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
-        inputs:
-          SourceFolder: '$(Build.BinariesDirectory)'
-          Contents: 'Release/dist/*.whl'
-          TargetFolder: '$(Build.ArtifactStagingDirectory)'
-
-      - task: CmdLine@2
-        displayName: 'Build Python Documentation'
-        condition: and(succeeded(), ne(variables['PythonVersion'], '3.9'))  # tensorflow not available on python 3.9
-        inputs:
-          script: |
-            mkdir -p $HOME/.onnx
-            docker run --rm \
-              --device=/dev/kfd \
-              --device=/dev/dri \
-              --group-add $(video) \
-              --group-add $(render) \
-              --privileged \
-              --ipc=host \
-              --network=host \
-              --cap-add=SYS_PTRACE \
-              --security-opt seccomp=unconfined \
-              --volume $(Build.SourcesDirectory):/onnxruntime_src \
-              --volume $(Build.BinariesDirectory):/build \
-              --entrypoint /bin/bash \
-              -e HIP_VISIBLE_DEVICES \
-              -e NIGHTLY_BUILD \
-              -e BUILD_BUILDNUMBER \
-              -e PythonManylinuxDir=$(PythonManylinuxdir) \
-              onnxruntimetrainingrocmbuild-torch$(TorchVersion)-rocm$(RocmVersion) \
-                /onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh
-          workingDirectory: $(Build.SourcesDirectory)
-
-      - task: CopyFiles@2
-        displayName: 'Copy Python Documentation to: $(Build.ArtifactStagingDirectory)'
-        condition: and(succeeded(), ne(variables['PythonVersion'], '3.9'))  # tensorflow not available on python 3.9
-        inputs:
-          SourceFolder: '$(Build.BinariesDirectory)/docs/training/html'
-          Contents: '**'
-          TargetFolder: '$(Build.ArtifactStagingDirectory)/training_html_doc'
-
-      - task: PublishBuildArtifacts@1
-        displayName: 'Upload Rocm wheel as build artifact'
-        inputs:
-          ArtifactName: onnxruntime_rocm
-
-      - script: |
-          python3 -m pip install azure-storage-blob==2.1.0
-          files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
-          echo ${files[0]} && \
-          python3 tools/ci_build/upload_python_package_to_azure_storage.py \
-              --python_wheel_path ${files[0]} \
-              --account_name onnxruntimepackages \
-              --account_key $(orttrainingpackagestorageaccountkey) \
-              --container_name '$web'
-        condition: and(succeeded(), eq(variables['DRY_RUN'], '0'))
-        displayName: 'Upload Rocm wheel to release repository'
-
-      - template: component-governance-component-detection-steps.yml
-        parameters:
-          condition: 'succeeded'
-
-      - template: clean-agent-build-directory-step.yml
-
   - ${{ if eq(parameters.enable_windows_cpu, true) }}:
     - job: Windows_py_Wheels
       pool: 'Win-CPU-2021'
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
index 6cc5fc6305..85dd578255 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
@@ -206,22 +206,16 @@ stages:
         inputs:
           ArtifactName: onnxruntime_gpu
 
-      - task: AzureCLI@2
+      - task: CmdLine@2
+        displayName: 'Upload wheel'
         condition: and(succeeded(), eq(variables['UploadWheel'], 'yes'))
         inputs:
-          azureSubscription: 'AIInfraBuildOnnxRuntimeOSS'
-          scriptType: 'bash'
-          scriptLocation: 'inlineScript'
-          inlineScript: |
-            python3 -m pip install azure-storage-blob==2.1.0
+          script: |
+            set -e -x
             files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
             echo ${files[0]} && \
             python3 tools/ci_build/upload_python_package_to_azure_storage.py \
-                --python_wheel_path ${files[0]} \
-                --account_name onnxruntimepackages \
-                --account_key $(orttrainingpackagestorageaccountkey) \
-                --container_name '$web'
-          displayName:
+                --python_wheel_path ${files[0]}
 
       - template: component-governance-component-detection-steps.yml
         parameters:
diff --git a/tools/ci_build/upload_python_package_to_azure_storage.py b/tools/ci_build/upload_python_package_to_azure_storage.py
index da7e4a8ea5..a0b3ad5836 100755
--- a/tools/ci_build/upload_python_package_to_azure_storage.py
+++ b/tools/ci_build/upload_python_package_to_azure_storage.py
@@ -5,7 +5,10 @@
 import os
 import argparse
 import warnings
-from azure.storage.blob import BlockBlobService, ContentSettings
+import subprocess
+import logging
+
+log = logging.getLogger("Build")
 
 
 def parse_nightly_and_local_version_from_whl_name(blob_name):
@@ -21,14 +24,14 @@ def parse_nightly_and_local_version_from_whl_name(blob_name):
     return night_build, blob_name[start:end]
 
 
-def upload_whl(python_wheel_path, account_name, account_key, container_name):
-    block_blob_service = BlockBlobService(
-        account_name=account_name,
-        account_key=account_key
-    )
+def run_subprocess(args, cwd=None):
+    log.debug("Running subprocess in '{0}'\n{1}".format(cwd or os.getcwd(), args))
+    return subprocess.run(args, cwd=cwd, check=True)
 
+
+def upload_whl(python_wheel_path):
     blob_name = os.path.basename(python_wheel_path)
-    block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path)
+    run_subprocess(['azcopy', 'cp', python_wheel_path, 'https://onnxruntimepackages.blob.core.windows.net/$web/'])
 
     nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name)
     if local_version:
@@ -38,7 +41,8 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name):
 
     download_path_to_html = "./onnxruntime_{}.html".format(nightly_build)
 
-    block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html)
+    run_subprocess(['azcopy', 'cp', 'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name,
+                    download_path_to_html])
 
     blob_name_plus_replaced = blob_name.replace('+', '%2B')
     with open(download_path_to_html) as f:
@@ -54,25 +58,16 @@ def upload_whl(python_wheel_path, account_name, account_key, container_name):
                 f.write("%s\n" % item)
     else:
         warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html))
-
-    content_settings = ContentSettings(content_type='text/html')
-    block_blob_service.create_blob_from_path(
-        container_name,
-        html_blob_name,
-        download_path_to_html,
-        content_settings=content_settings)
+    run_subprocess(['azcopy', 'cp', download_path_to_html,
+                    'https://onnxruntimepackages.blob.core.windows.net/$web/'+html_blob_name,
+                    '--content-type', 'text/html', '--overwrite', 'true'])
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Upload python whl to azure storage.")
 
     parser.add_argument("--python_wheel_path", type=str, help="path to python wheel")
-    parser.add_argument("--account_name", type=str, help="account name")
-    parser.add_argument("--account_key", type=str, help="account key")
-    parser.add_argument("--container_name", type=str, help="container name")
 
-    # TODO: figure out a way to secure args.account_key to prevent later code changes
-    # that may accidentally print out it to the console.
     args = parser.parse_args()
 
-    upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name)
+    upload_whl(args.python_wheel_path)
diff --git a/tools/ci_build/upload_python_package_to_azure_storage_with_python.py b/tools/ci_build/upload_python_package_to_azure_storage_with_python.py
new file mode 100644
index 0000000000..da7e4a8ea5
--- /dev/null
+++ b/tools/ci_build/upload_python_package_to_azure_storage_with_python.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+import argparse
+import warnings
+from azure.storage.blob import BlockBlobService, ContentSettings
+
+
+def parse_nightly_and_local_version_from_whl_name(blob_name):
+    night_build = 'nightly' if blob_name.find(".dev") > 0 else 'stable'
+
+    start = blob_name.find("+")
+    if start == -1:
+        return night_build, None
+    start = start + 1
+    end = blob_name.find("-", start)
+    if end == -1:
+        return night_build, None
+    return night_build, blob_name[start:end]
+
+
+def upload_whl(python_wheel_path, account_name, account_key, container_name):
+    block_blob_service = BlockBlobService(
+        account_name=account_name,
+        account_key=account_key
+    )
+
+    blob_name = os.path.basename(python_wheel_path)
+    block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path)
+
+    nightly_build, local_version = parse_nightly_and_local_version_from_whl_name(blob_name)
+    if local_version:
+        html_blob_name = 'onnxruntime_{}_{}.html'.format(nightly_build, local_version)
+    else:
+        html_blob_name = 'onnxruntime_{}.html'.format(nightly_build)
+
+    download_path_to_html = "./onnxruntime_{}.html".format(nightly_build)
+
+    block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html)
+
+    blob_name_plus_replaced = blob_name.replace('+', '%2B')
+    with open(download_path_to_html) as f:
+        lines = f.read().splitlines()
+
+    new_line = '<a href="{blobname}">{blobname}</a><br>'.format(blobname=blob_name_plus_replaced)
+    if new_line not in lines:
+        lines.append(new_line)
+        lines.sort()
+
+        with open(download_path_to_html, 'w') as f:
+            for item in lines:
+                f.write("%s\n" % item)
+    else:
+        warnings.warn("'{}' exists in {}. The html file is not updated.".format(new_line, download_path_to_html))
+
+    content_settings = ContentSettings(content_type='text/html')
+    block_blob_service.create_blob_from_path(
+        container_name,
+        html_blob_name,
+        download_path_to_html,
+        content_settings=content_settings)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Upload python whl to azure storage.")
+
+    parser.add_argument("--python_wheel_path", type=str, help="path to python wheel")
+    parser.add_argument("--account_name", type=str, help="account name")
+    parser.add_argument("--account_key", type=str, help="account key")
+    parser.add_argument("--container_name", type=str, help="container name")
+
+    # TODO: figure out a way to secure args.account_key to prevent later code changes
+    # that may accidentally print out it to the console.
+    args = parser.parse_args()
+
+    upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name)