diff --git a/setup.py b/setup.py
index 4bcf02b395..4ac1fbe6da 100644
--- a/setup.py
+++ b/setup.py
@@ -19,52 +19,54 @@ featurizers_build = False
 package_name = 'onnxruntime'
 wheel_name_suffix = None
 
-# Any combination of the following arguments can be applied
-if '--use_featurizers' in sys.argv:
-    featurizers_build = True
-    sys.argv.remove('--use_featurizers')
+def parse_arg_remove_boolean(argv, arg_name):
+    arg_value = False
+    if arg_name in sys.argv:
+        arg_value = True
+        argv.remove(arg_name)
 
-if '--nightly_build' in sys.argv:
+    return arg_value
+
+def parse_arg_remove_string(argv, arg_name_equal):
+    arg_value = None
+    for arg in sys.argv[1:]:
+        if arg.startswith(arg_name_equal):
+            arg_value = arg[len(arg_name_equal):]
+            sys.argv.remove(arg)
+            break
+
+    return arg_value
+
+# Any combination of the following arguments can be applied
+featurizers_build = parse_arg_remove_boolean(sys.argv, '--use_featurizers')
+
+if parse_arg_remove_boolean(sys.argv, '--nightly_build'):
     package_name = 'ort-nightly'
     nightly_build = True
-    sys.argv.remove('--nightly_build')
 
-for arg in sys.argv[1:]:
-    if arg.startswith("--wheel_name_suffix="):
-        wheel_name_suffix = arg[len("--wheel_name_suffix="):]
-
-        sys.argv.remove(arg)
-
-        break
+wheel_name_suffix = parse_arg_remove_string(sys.argv, '--wheel_name_suffix=')
 
+cuda_version = None
 # The following arguments are mutually exclusive
-if '--use_tensorrt' in sys.argv:
+if parse_arg_remove_boolean(sys.argv, '--use_tensorrt'):
     package_name = 'onnxruntime-gpu-tensorrt' if not nightly_build else 'ort-trt-nightly'
-    sys.argv.remove('--use_tensorrt')
-elif '--use_cuda' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_cuda'):
     package_name = 'onnxruntime-gpu' if not nightly_build else 'ort-gpu-nightly'
-    sys.argv.remove('--use_cuda')
-elif '--use_openvino' in sys.argv:
+    cuda_version = parse_arg_remove_string(sys.argv, '--cuda_version=')
+elif parse_arg_remove_boolean(sys.argv, '--use_openvino'):
     package_name = 'onnxruntime-openvino'
-    sys.argv.remove('--use_openvino')
-elif '--use_dnnl' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_dnnl'):
     package_name = 'onnxruntime-dnnl'
-    sys.argv.remove('--use_dnnl')
-elif '--use_nuphar' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_nuphar'):
     package_name = 'onnxruntime-nuphar'
-    sys.argv.remove('--use_nuphar')
-elif '--use_vitisai' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_vitisai'):
     package_name = 'onnxruntime-vitisai'
-    sys.argv.remove('--use_vitisai')
-elif '--use_acl' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_acl'):
     package_name = 'onnxruntime-acl'
-    sys.argv.remove('--use_acl')
-elif '--use_armnn' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_armnn'):
     package_name = 'onnxruntime-armnn'
-    sys.argv.remove('--use_armnn')
-elif '--use_dml' in sys.argv:
+elif parse_arg_remove_boolean(sys.argv, '--use_dml'):
     package_name = 'onnxruntime-dml'
-    sys.argv.remove('--use_dml')
 
 # PEP 513 defined manylinux1_x86_64 and manylinux1_i686
 # PEP 571 defined manylinux2010_x86_64 and manylinux2010_i686
@@ -234,12 +236,25 @@ packages = [
 
 requirements_file = "requirements.txt"
 
-if '--enable_training' in sys.argv:
+local_version = None
+enable_training = parse_arg_remove_boolean(sys.argv, '--enable_training')
+if enable_training:
     packages.extend(['onnxruntime.training',
                      'onnxruntime.training.amp',
                      'onnxruntime.training.optim'])
-    sys.argv.remove('--enable_training')
     requirements_file = "requirements-training.txt"
+    # with training, we want to follow this naming convention:
+    # stable:
+    # onnxruntime-training-1.7.0+cu111-cp36-cp36m-linux_x86_64.whl
+    # nightly:
+    # onnxruntime-training-1.7.0.dev20210408+cu111-cp36-cp36m-linux_x86_64.whl
+    # this is needed immediately by pytorch/ort so that the user is able to
+    # install an onnxruntime training package with matching torch cuda version.
+    package_name = 'onnxruntime-training'
+    if cuda_version and not nightly_build:
+        # removing '.' to make Cuda version number in the same form as Pytorch.
+        cuda_version = cuda_version.replace('.', '')
+        local_version = '+cu' + cuda_version
 
 package_data = {}
 data_files = []
@@ -295,21 +310,38 @@ version_number = ''
 with open('VERSION_NUMBER') as f:
     version_number = f.readline().strip()
 if nightly_build:
-    #https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables
+    # https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables
     build_suffix = environ.get('BUILD_BUILDNUMBER')
     if build_suffix is None:
-      #The following line is only for local testing
-      build_suffix = str(datetime.datetime.now().date().strftime("%Y%m%d"))
+        # The following line is only for local testing
+        build_suffix = str(datetime.datetime.now().date().strftime("%Y%m%d"))
     else:
-      build_suffix = build_suffix.replace('.','')
+        build_suffix = build_suffix.replace('.', '')
+
+    if enable_training:
+        from packaging import version
+        from packaging.version import Version
+        # with training package, we need to bump up version minor number so that
+        # nightly releases take precedence over the latest release when --pre is used during pip install.
+        # eventually this shall be the behavior of all onnxruntime releases.
+        # alternatively we may bump up version number right after every release.
+        ort_version = version.parse(version_number)
+        if isinstance(ort_version, Version):
+            version_number = '{major}.{minor}.{macro}'.format(
+                major=ort_version.major,
+                minor=ort_version.minor + 1,
+                macro=ort_version.micro)
 
     version_number = version_number + ".dev" + build_suffix
 
+if local_version:
+    version_number = version_number + local_version
+
 if wheel_name_suffix:
     package_name = "{}_{}".format(package_name, wheel_name_suffix)
 
 cmd_classes = {}
-if bdist_wheel is not None :
+if bdist_wheel is not None:
     cmd_classes['bdist_wheel'] = bdist_wheel
 cmd_classes['build_ext'] = build_ext
 
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index 7aa77a3363..ad72044afd 100644
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -1474,7 +1474,7 @@ def run_nodejs_tests(nodejs_binding_dir):
 
 
 def build_python_wheel(
-        source_dir, build_dir, configs, use_cuda, use_dnnl,
+        source_dir, build_dir, configs, use_cuda, cuda_version, use_dnnl,
         use_tensorrt, use_openvino, use_nuphar, use_vitisai, use_acl, use_armnn, use_dml,
         wheel_name_suffix, enable_training, nightly_build=False, featurizers_build=False, use_ninja=False):
     for config in configs:
@@ -1510,6 +1510,8 @@ def build_python_wheel(
             args.append('--use_tensorrt')
         elif use_cuda:
             args.append('--use_cuda')
+            if cuda_version:
+                args.append('--cuda_version={}'.format(cuda_version))
         elif use_openvino:
             args.append('--use_openvino')
         elif use_dnnl:
@@ -2019,6 +2021,7 @@ def main():
                 build_dir,
                 configs,
                 args.use_cuda,
+                args.cuda_version,
                 args.use_dnnl,
                 args.use_tensorrt,
                 args.use_openvino,
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml
index c8ad072f1d..c5568e5381 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml
@@ -3,7 +3,7 @@ trigger: none
 stages:
 - template: templates/py-packaging-stage.yml
   parameters:
-    build_py_parameters: --enable_training --wheel_name_suffix=training
+    build_py_parameters: --enable_training
     enable_linux_cpu: false
     enable_linux_gpu: false
     enable_linux_gpu_training: true
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
index f292a4d74a..b22d1967d6 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -235,6 +235,7 @@ stages:
           # Python39:
           #   PythonVersion: '3.9'
       steps:
+
       - checkout: self
         clean: true
         submodules: recursive
@@ -276,7 +277,7 @@ stages:
                   --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc  PYTHON_INCLUDE_DIR=$(PythonManylinuxIncludeDir) PYTHON_LIBRARY=/usr/lib64/librt.so \
                   --use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1
           workingDirectory: $(Build.SourcesDirectory)
-
+  
       - task: CopyFiles@2
         displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
         inputs:
@@ -321,27 +322,53 @@ stages:
         displayName: 'sudo apt-get install python3-pip python-dev'
 
       - script: |
-          python3 -m pip install twine
-        displayName: 'python3 -m pip install twine'
+          python3 -m pip install azure-storage-blob==2.1.0
+        displayName: 'python3 -m pip install azure-storage-blob==2.1.0'
         timeoutInMinutes: 20
 
-      # this block does not work because TwineAuthenticate@1 will trigger cleanup of $(PYPIRC_PATH)
-      # at the end of pipeline execution. Because $(PYPIRC_PATH) is already cleaned by clean-agent-build-directory-step.yml
-      # the cleanup task for TwineAuthenticate@1 will fail. For this reason, we cannot used TwineAuthenticate@1.
-      # - task: TwineAuthenticate@1
-      #   inputs:
-      #     artifactFeed: 'lotus/ort-gpu-nightly-training-feed'
+      - task: AzureCLI@2
+        inputs:
+          azureSubscription: 'AIInfraBuildOnnxRuntimeOSS'
+          scriptType: 'bash'
+          scriptLocation: 'inlineScript'
+          inlineScript: |
+            files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
+            echo ${files[0]} && \
+            tools/ci_build/upload_python_package_to_azure_storage.py \
+                --python_wheel_path ${files[0]} \
+                --account_name onnxruntimepackages \
+                --account_key $(orttrainingpackagestorageaccountkey) \
+                --container_name '$web'
+          condition: succeededOrFailed()
+          displayName: 
 
       # - script: |
-      #     python3 -m twine upload -r ort-gpu-nightly-training-feed --config-file $(PYPIRC_PATH) $(Build.ArtifactStagingDirectory)/Release/dist/*.whl
-      #   displayName: 'python3 -m twine upload -r ort-gpu-nightly-training-feed $(Build.ArtifactStagingDirectory)/Release/dist/*.whl'
+      #     sudo apt-get update
+      #     sudo apt-get install python3-pip python-dev
+      #   displayName: 'sudo apt-get install python3-pip python-dev'
+
+      # - script: |
+      #     python3 -m pip install twine
+      #   displayName: 'python3 -m pip install twine'
       #   timeoutInMinutes: 20
 
-      - script: |
-          python3 -m twine upload -r ORT-Nightly --repository-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/upload \
-          --username $(ortpypitrainingnightlyusername) --password $(aiinfrapypifeedpassword) $(Build.ArtifactStagingDirectory)/Release/dist/*.whl
-        displayName: 'python3 -m twine upload $(Build.ArtifactStagingDirectory)/Release/dist/*.whl'
-        timeoutInMinutes: 20
+      # # this block does not work because TwineAuthenticate@1 will trigger cleanup of $(PYPIRC_PATH)
+      # # at the end of pipeline execution. Because $(PYPIRC_PATH) is already cleaned by clean-agent-build-directory-step.yml
+      # # the cleanup task for TwineAuthenticate@1 will fail. For this reason, we cannot used TwineAuthenticate@1.
+      # # - task: TwineAuthenticate@1
+      # #   inputs:
+      # #     artifactFeed: 'lotus/ort-gpu-nightly-training-feed'
+
+      # # - script: |
+      # #     python3 -m twine upload -r ort-gpu-nightly-training-feed --config-file $(PYPIRC_PATH) $(Build.ArtifactStagingDirectory)/Release/dist/*.whl
+      # #   displayName: 'python3 -m twine upload -r ort-gpu-nightly-training-feed $(Build.ArtifactStagingDirectory)/Release/dist/*.whl'
+      # #   timeoutInMinutes: 20
+
+      # - script: |
+      #     python3 -m twine upload -r ORT-Nightly --repository-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/upload \
+      #     --username $(ortpypitrainingnightlyusername) --password $(aiinfrapypifeedpassword) $(Build.ArtifactStagingDirectory)/Release/dist/*.whl
+      #   displayName: 'python3 -m twine upload $(Build.ArtifactStagingDirectory)/Release/dist/*.whl'
+      #   timeoutInMinutes: 20
 
       - template: component-governance-component-detection-steps.yml
         parameters:
diff --git a/tools/ci_build/upload_python_package_to_azure_storage.py b/tools/ci_build/upload_python_package_to_azure_storage.py
new file mode 100755
index 0000000000..c96bfbc205
--- /dev/null
+++ b/tools/ci_build/upload_python_package_to_azure_storage.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+import argparse
+from azure.storage.blob import BlockBlobService, ContentSettings
+
+
+def upload_whl(python_wheel_path, account_name, account_key, container_name):
+    block_blob_service = BlockBlobService(
+        account_name=account_name,
+        account_key=account_key
+    )
+
+    blob_name = os.path.basename(python_wheel_path)
+    block_blob_service.create_blob_from_path(container_name, blob_name, python_wheel_path)
+
+    html_blob_name = 'onnxruntime_nightly.html'
+    download_path_to_html = "./onnxruntime_nightly.html"
+    block_blob_service.get_blob_to_path(container_name, html_blob_name, download_path_to_html)
+
+    blob_name_plus_replaced = blob_name.replace('+', '%2B')
+    with open(download_path_to_html) as f:
+        lines = f.read().splitlines()
+
+    new_line = '<a href="{blobname}">{blobname}</a><br>'.format(blobname=blob_name_plus_replaced)
+    lines.append(new_line)
+    lines.sort()
+
+    with open(download_path_to_html, 'w') as f:
+        for item in lines:
+            f.write("%s\n" % item)
+
+    content_settings = ContentSettings(content_type='text/html')
+    block_blob_service.create_blob_from_path(
+        container_name,
+        html_blob_name,
+        download_path_to_html,
+        content_settings=content_settings)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Upload python whl to azure storage.")
+
+    parser.add_argument("--python_wheel_path", type=str, help="path to python wheel")
+    parser.add_argument("--account_name", type=str, help="account name")
+    parser.add_argument("--account_key", type=str, help="account key")
+    parser.add_argument("--container_name", type=str, help="container name")
+
+    # TODO: figure out a way to secure args.account_key to prevent later code changes
+    # that may accidentally print out it to the console.
+    args = parser.parse_args()
+
+    upload_whl(args.python_wheel_path, args.account_name, args.account_key, args.container_name)