diff --git a/orttraining/orttraining/test/python/_orttraining_ortmodule_models.py b/orttraining/orttraining/test/python/_orttraining_ortmodule_models.py
index f1e5c22bfa..12fa318c54 100644
--- a/orttraining/orttraining/test/python/_orttraining_ortmodule_models.py
+++ b/orttraining/orttraining/test/python/_orttraining_ortmodule_models.py
@@ -54,26 +54,3 @@ class NeuralNetCustomClassOutput(torch.nn.Module):
         out2 = self.fc2_2(self.relu2(self.fc2_1(input2)))
         out3 = self.fc3_2(self.relu3(self.fc3_1(input3)))
         return NeuralNetCustomClassOutput.CustomClass(out1, out2, out3)
-
-
-class MyCustomFunctionReluModel(torch.nn.Module):
-    def __init__(self):
-        super().__init__()
-
-        class MyReLU(torch.autograd.Function):
-            @staticmethod
-            def forward(ctx, input):
-                ctx.save_for_backward(input)
-                return input.clamp(min=0)
-
-            @staticmethod
-            def backward(ctx, grad_output):
-                (input,) = ctx.saved_tensors
-                grad_input = grad_output.clone()
-                grad_input[input < 0] = 0
-                return grad_input
-
-        self.relu = MyReLU.apply
-
-    def forward(self, input):
-        return self.relu(input)
diff --git a/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py b/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py
index 0c8a24d20e..d07e0aa824 100644
--- a/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py
+++ b/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py
@@ -142,6 +142,14 @@ def run_data_sampler_tests(cwd, log):
     run_subprocess(command, cwd=cwd, log=log).check_returncode()
 
 
+def run_pytorch_export_contrib_ops_tests(cwd, log):
+    log.debug("Running: PyTorch Export Contrib Ops Tests")
+
+    command = [sys.executable, "-m", "pytest", "-sv", "test_pytorch_export_contrib_ops.py"]
+
+    run_subprocess(command, cwd=cwd, log=log).check_returncode()
+
+
 def main():
     args = parse_arguments()
     cwd = args.cwd
@@ -178,6 +186,9 @@ def main():
 
     run_experimental_gradient_graph_tests(cwd, log)
 
+    # TODO(bmeswani): Enable this test once it can run with latest pytorch
+    # run_pytorch_export_contrib_ops_tests(cwd, log)
+
     return 0
 
 
diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py
index 554184ef7d..7758603c48 100644
--- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py
+++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py
@@ -4820,9 +4820,14 @@ def test_ortmodule_attribute_name_collision_warning():
     with pytest.warns(UserWarning) as warning_record:
         ort_model = ORTModule(pt_model)
 
-    assert len(warning_record) == 2
-    assert "_torch_module collides with ORTModule's attribute name." in warning_record[0].message.args[0]
-    assert "load_state_dict collides with ORTModule's attribute name." in warning_record[1].message.args[0]
+    # FutureWarning('The first argument to symbolic functions is deprecated in 1.13 and will be removed in the future.
+    # Please annotate treat the first argument (g) as GraphContext and use context information from the object
+    # instead.')
+    # TODO(bmeswani): Check with the exporter team as to what this might mean for ortmodule.
+    assert len(warning_record) == 3
+
+    assert "_torch_module collides with ORTModule's attribute name." in warning_record[1].message.args[0]
+    assert "load_state_dict collides with ORTModule's attribute name." in warning_record[2].message.args[0]
 
 
 def test_ortmodule_ortmodule_method_attribute_copy():
diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py
index 1ee2cec568..6cde304a65 100644
--- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py
+++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py
@@ -14,7 +14,6 @@ import pytest
 import torch
 from _orttraining_ortmodule_models import (
     MyCustomClassInputNet,
-    MyCustomFunctionReluModel,
     NeuralNetCustomClassOutput,
     NeuralNetSinglePositionalArgument,
 )
@@ -482,75 +481,6 @@ def test_ortmodule_fallback_init__missing_cpp_extensions(
                 assert "ORTModule's extensions were not detected" in str(ex_info.value)
 
 
-@pytest.mark.parametrize(
-    "is_training,fallback_enabled,matching_policy,persist_fallback", list(itertools.product([True, False], repeat=4))
-)
-def test_ortmodule_fallback_onnx_model__custom_autograd(
-    is_training, fallback_enabled, matching_policy, persist_fallback
-):
-    from onnxruntime.training.ortmodule._custom_autograd_function import (
-        custom_autograd_function_enabler,
-        enable_custom_autograd_support,
-    )
-
-    # Disable the autograd support to test the fallback.
-    old_state = custom_autograd_function_enabler.state
-    enable_custom_autograd_support(False)
-
-    # is_training: True for torch.nn.Module training model, eval mode otherwise
-    # fallback_enabled: True PyTorch executes the forward graph instead of ORT backend
-    # matching_policy: True matches FALLBACK_UNSUPPORTED_ONNX_MODEL policy to ORTModuleDeviceException exception.
-    #   Otherwise, an incorrect policy (FALLBACK_UNSUPPORTED_DEVICE) is used to verify that the fallback does not happen
-
-    if fallback_enabled:
-        if matching_policy:
-            policy = "FALLBACK_UNSUPPORTED_ONNX_MODEL"
-        else:
-            policy = "FALLBACK_UNSUPPORTED_DEVICE"
-    else:
-        policy = "FALLBACK_DISABLE"
-    os.environ["ORTMODULE_FALLBACK_POLICY"] = policy
-    os.environ["ORTMODULE_FALLBACK_RETRY"] = str(not persist_fallback)
-
-    dtype = torch.float
-    device = torch.device("cuda")
-    N, D_in, H, D_out = 64, 1000, 100, 10
-
-    x = torch.randn(N, D_in, device=device, dtype=dtype)
-    y = torch.randn(N, D_out, device=device, dtype=dtype)
-    w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)
-    w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)
-
-    pt_model = MyCustomFunctionReluModel()
-    ort_model = ORTModule(copy.deepcopy(pt_model))
-    ort_model.train(is_training)
-    pt_model.train(is_training)
-
-    for i in range(3):
-        if fallback_enabled:
-            if matching_policy:
-                if i > 0 and persist_fallback:
-                    assert (
-                        ort_model._torch_module._execution_manager(is_training=is_training)._fallback_manager._exception
-                        is not None
-                    )
-                pt_out = pt_model(x.mm(w1)).mm(w2)
-                ort_out = ort_model(x.mm(w1)).mm(w2)
-                _test_helpers.assert_values_are_close(ort_out, pt_out, rtol=1e-03, atol=1e-04)
-            else:
-                with pytest.raises(_fallback.ORTModuleONNXModelException) as ex_info:
-                    _ = ort_model(x.mm(w1)).mm(w2)
-                assert "There was an error while exporting the PyTorch model to ONNX" in str(ex_info.value)
-        else:
-            with pytest.raises(_fallback.ORTModuleONNXModelException) as ex_info:
-                # Initialize with fallback policy because Exception will happen during __init__
-                _ = ort_model(x.mm(w1)).mm(w2)
-            assert "There was an error while exporting the PyTorch model to ONNX" in str(ex_info.value)
-
-    # Restore the autograd support state.
-    enable_custom_autograd_support(old_state)
-
-
 @pytest.mark.parametrize(
     "is_training,fallback_enabled,matching_policy,persist_fallback", list(itertools.product([True, False], repeat=4))
 )
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index 90e63ebfe4..98af43b403 100644
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -1667,168 +1667,6 @@ def run_ios_tests(args, source_dir, config, cwd):
         )
 
 
-def run_orttraining_test_orttrainer_frontend_separately(cwd):
-    class TestNameCollecterPlugin:
-        def __init__(self):
-            self.collected = set()
-
-        def pytest_collection_modifyitems(self, items):
-            for item in items:
-                print("item.name: ", item.name)
-                test_name = item.name
-                start = test_name.find("[")
-                if start > 0:
-                    test_name = test_name[:start]
-                self.collected.add(test_name)
-
-    import pytest
-
-    plugin = TestNameCollecterPlugin()
-    test_script_filename = os.path.join(cwd, "orttraining_test_orttrainer_frontend.py")
-    pytest.main(["--collect-only", test_script_filename], plugins=[plugin])
-
-    for test_name in plugin.collected:
-        run_subprocess(
-            [sys.executable, "-m", "pytest", "orttraining_test_orttrainer_frontend.py", "-v", "-k", test_name], cwd=cwd
-        )
-
-
-def run_training_python_frontend_tests(cwd):
-    # have to disable due to (with torchvision==0.9.1+cu102 which is required by ortmodule):
-    # Downloading http://yann.lecun.com/exdb/mnist/
-    # https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz
-    # Failed to download (trying next):
-    # HTTP Error 404: Not Found
-    # run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer.py'], cwd=cwd)
-
-    run_subprocess([sys.executable, "onnxruntime_test_training_unit_tests.py"], cwd=cwd)
-    run_subprocess(
-        [
-            sys.executable,
-            "orttraining_test_transformers.py",
-            "BertModelTest.test_for_pretraining_full_precision_list_input",
-        ],
-        cwd=cwd,
-    )
-    run_subprocess(
-        [
-            sys.executable,
-            "orttraining_test_transformers.py",
-            "BertModelTest.test_for_pretraining_full_precision_dict_input",
-        ],
-        cwd=cwd,
-    )
-    run_subprocess(
-        [
-            sys.executable,
-            "orttraining_test_transformers.py",
-            "BertModelTest.test_for_pretraining_full_precision_list_and_dict_input",
-        ],
-        cwd=cwd,
-    )
-
-    # TODO: use run_orttraining_test_orttrainer_frontend_separately to work around a sporadic segfault.
-    # shall revert to run_subprocess call once the segfault issue is resolved.
-    run_orttraining_test_orttrainer_frontend_separately(cwd)
-    # run_subprocess([sys.executable, '-m', 'pytest', '-sv', 'orttraining_test_orttrainer_frontend.py'], cwd=cwd)
-
-    run_subprocess([sys.executable, "-m", "pytest", "-sv", "orttraining_test_orttrainer_bert_toy_onnx.py"], cwd=cwd)
-
-    run_subprocess([sys.executable, "-m", "pytest", "-sv", "orttraining_test_checkpoint_storage.py"], cwd=cwd)
-
-    run_subprocess(
-        [sys.executable, "-m", "pytest", "-sv", "orttraining_test_orttrainer_checkpoint_functions.py"], cwd=cwd
-    )
-    # Not technically training related, but it needs torch to be installed.
-    run_subprocess([sys.executable, "-m", "pytest", "-sv", "test_pytorch_export_contrib_ops.py"], cwd=cwd)
-
-
-def run_training_python_frontend_e2e_tests(cwd):
-    # frontend tests are to be added here:
-    log.info("Running python frontend e2e tests.")
-
-    run_subprocess(
-        [sys.executable, "orttraining_run_frontend_batch_size_test.py", "-v"],
-        cwd=cwd,
-        env={"CUDA_VISIBLE_DEVICES": "0"},
-    )
-
-    import torch
-
-    ngpus = torch.cuda.device_count()
-    if ngpus > 1:
-        bert_pretrain_script = "orttraining_run_bert_pretrain.py"
-        # TODO: this test will be replaced with convergence test ported from backend
-        log.debug(
-            "RUN: mpirun -n {} "
-            "-x"
-            "NCCL_DEBUG=INFO"
-            " {} {} {}".format(
-                ngpus, sys.executable, bert_pretrain_script, "ORTBertPretrainTest.test_pretrain_convergence"
-            )
-        )
-        run_subprocess(
-            [
-                "mpirun",
-                "-n",
-                str(ngpus),
-                "-x",
-                "NCCL_DEBUG=INFO",
-                sys.executable,
-                bert_pretrain_script,
-                "ORTBertPretrainTest.test_pretrain_convergence",
-            ],
-            cwd=cwd,
-        )
-
-        log.debug("RUN: mpirun -n {} {} orttraining_run_glue.py".format(ngpus, sys.executable))
-        run_subprocess(
-            ["mpirun", "-n", str(ngpus), "-x", "NCCL_DEBUG=INFO", sys.executable, "orttraining_run_glue.py"], cwd=cwd
-        )
-
-    # with orttraining_run_glue.py.
-    # 1. we like to force to use single GPU (with CUDA_VISIBLE_DEVICES)
-    #   for fine-tune tests.
-    # 2. need to run test separately (not to mix between fp16
-    #   and full precision runs. this need to be investigated).
-    run_subprocess(
-        [sys.executable, "orttraining_run_glue.py", "ORTGlueTest.test_bert_with_mrpc", "-v"],
-        cwd=cwd,
-        env={"CUDA_VISIBLE_DEVICES": "0"},
-    )
-
-    run_subprocess(
-        [sys.executable, "orttraining_run_glue.py", "ORTGlueTest.test_bert_fp16_with_mrpc", "-v"],
-        cwd=cwd,
-        env={"CUDA_VISIBLE_DEVICES": "0"},
-    )
-
-    run_subprocess(
-        [sys.executable, "orttraining_run_glue.py", "ORTGlueTest.test_roberta_with_mrpc", "-v"],
-        cwd=cwd,
-        env={"CUDA_VISIBLE_DEVICES": "0"},
-    )
-
-    run_subprocess(
-        [sys.executable, "orttraining_run_glue.py", "ORTGlueTest.test_roberta_fp16_with_mrpc", "-v"],
-        cwd=cwd,
-        env={"CUDA_VISIBLE_DEVICES": "0"},
-    )
-
-    run_subprocess(
-        [sys.executable, "orttraining_run_multiple_choice.py", "ORTMultipleChoiceTest.test_bert_fp16_with_swag", "-v"],
-        cwd=cwd,
-        env={"CUDA_VISIBLE_DEVICES": "0"},
-    )
-
-    run_subprocess([sys.executable, "onnxruntime_test_ort_trainer_with_mixed_precision.py"], cwd=cwd)
-
-    run_subprocess(
-        [sys.executable, "orttraining_test_transformers.py", "BertModelTest.test_for_pretraining_mixed_precision"],
-        cwd=cwd,
-    )
-
-
 def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
     for config in configs:
         log.info("Running tests for %s configuration", config)
@@ -1947,11 +1785,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
             if not args.disable_ml_ops and not args.use_tensorrt:
                 run_subprocess([sys.executable, "onnxruntime_test_python_mlops.py"], cwd=cwd, dll_path=dll_path)
 
-            # The following test has multiple failures on Windows
-            if args.enable_training and args.use_cuda and not is_windows():
-                # run basic frontend tests
-                run_training_python_frontend_tests(cwd=cwd)
-
             if args.build_eager_mode:
                 # run eager mode test
                 args_list = [sys.executable, os.path.join(cwd, "eager_test")]
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml
index e4364ee52b..6042184eb5 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml
@@ -9,6 +9,7 @@ jobs:
     RunDockerBuildArgs: >
       -o ubuntu20.04 -d gpu
       -t onnxruntime_orttraining_ortmodule_tests_image
+      -u
       -e
       -x "
       --enable_training
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
index 2fbfece8de..58f5d13cf6 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
@@ -12,7 +12,7 @@ stages:
 - template: templates/py-packaging-training-cuda-stage.yml
   parameters:
     build_py_parameters: --enable_training --update --build --enable_training_apis
-    torch_version: '1.11.0'
+    torch_version: '1.13.1'
     opset_version: '15'
     cuda_version: '11.6'
     gcc_version: 11
@@ -25,7 +25,7 @@ stages:
 - template: templates/py-packaging-training-cuda-stage.yml
   parameters:
     build_py_parameters: --enable_training --update --build --enable_training_apis
-    torch_version: '1.11.0'
+    torch_version: '1.13.1'
     opset_version: '15'
     cuda_version: '11.6'
     gcc_version: 11
diff --git a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml
index e8692fb933..68d6b4d4a3 100644
--- a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml
@@ -33,7 +33,7 @@ steps:
       --volume /bert_data:/bert_data \
       --volume /hf_models_cache:/hf_models_cache \
       ${{ parameters.DockerImageTag }} \
-        bash -c "python3 -m pip uninstall -y -r /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/training/requirements.txt && python3 -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.12.1_cu11.6/requirements.txt && python3 -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt && rm -rf /build/onnxruntime/ && python3 -m pip install /build/dist/onnxruntime*.whl && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && /build/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_tests.py --mnist /mnist --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw --transformers_cache /hf_models_cache/huggingface/transformers' --cwd /build" \
+        bash -c "rm -rf /build/onnxruntime/ && python3 -m pip install /build/dist/onnxruntime*.whl && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && /build/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_tests.py --mnist /mnist --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw --transformers_cache /hf_models_cache/huggingface/transformers' --cwd /build" \
   displayName: 'Run orttraining_ortmodule_tests.py'
   condition: succeededOrFailed()
   timeoutInMinutes: 60
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
index 72d196bb38..90d996c0d7 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
@@ -127,14 +127,6 @@ jobs:
           workingDirectory: '$(Build.BinariesDirectory)'
           arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }}
 
-
-      #Build.py disables running training python frontend tests on Windows, so here we don't install pytorch.
-      - ${{ if eq(parameters.isTraining, true) }}:
-          - script: |
-             python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\linux\docker\scripts\training\requirements.txt
-            workingDirectory: '$(Build.BinariesDirectory)'
-            displayName: 'Install python modules'
-
   - task: NuGetToolInstaller@0
     displayName: Use Nuget 5.7.0
     inputs:
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_6 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_6
index aac62219ac..fc7a6bc827 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_6
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_6
@@ -165,7 +165,7 @@ CMD ["/bin/bash"]
 
 #Build manylinux2014 docker image end
 ARG PYTHON_VERSION=3.9
-ARG TORCH_VERSION=1.12.1
+ARG TORCH_VERSION=1.13.1
 ARG OPSET_VERSION=15
 ARG INSTALL_DEPS_EXTRA_ARGS
 
diff --git a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh
index 42e167c987..86a8d7a63a 100755
--- a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh
+++ b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh
@@ -6,7 +6,7 @@ INSTALL_DEPS_DISTRIBUTED_SETUP=false
 ORTMODULE_BUILD=false
 TARGET_ROCM=false
 CU_VER="11.6"
-TORCH_VERSION='1.12.1'
+TORCH_VERSION='1.13.1'
 USE_CONDA=false
 
 while getopts p:h:d:v:tmurc parameter_Option
diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh
index 845828483e..276df0d75a 100755
--- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh
+++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh
@@ -96,7 +96,7 @@ do
   if ! [[ ${PYTHON_EXE} = "/opt/python/cp310-cp310/bin/python3.10" ]]; then
     ${PYTHON_EXE} -m pip install -r ${0/%install_deps_aten\.sh/..\/training\/ortmodule\/stage1\/requirements_torch_cpu\/requirements.txt}
   else
-    ${PYTHON_EXE} -m pip install torch==1.12.1
+    ${PYTHON_EXE} -m pip install torch==1.13.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
   fi
 done
 
diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_cu11.6/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_cu11.6/requirements.txt
deleted file mode 100644
index 8d087ed76b..0000000000
--- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_cu11.6/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
---pre
--f https://download.pytorch.org/whl/torch_stable.html
-torch==1.12.1+cu116
-torchvision==0.13.1+cu116
-torchtext==0.13.1
-setuptools>=41.4.0
diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.12.1_cu11.6/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.13.1_cu11.6/requirements.txt
similarity index 77%
rename from tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.12.1_cu11.6/requirements.txt
rename to tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.13.1_cu11.6/requirements.txt
index aa236bcdf6..131e5811d6 100644
--- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.12.1_cu11.6/requirements.txt
+++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.13.1_cu11.6/requirements.txt
@@ -1,8 +1,8 @@
 --pre
 -f https://download.pytorch.org/whl/torch_stable.html
-torch==1.12.1+cu116
-torchvision==0.13.1+cu116
-torchtext==0.13.1
+torch==1.13.1+cu116
+torchvision==0.14.1+cu116
+torchtext==0.14.1
 # TODO(bmeswani): packaging 22.0 removes support for LegacyVersion leading to errors because transformers 4.4.2 uses LegacyVersion
 packaging==21.3
 setuptools>=41.4.0
diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt
index 66d87a11fe..914fc3a95d 100644
--- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt
+++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt
@@ -1,2 +1,3 @@
-torch==1.12
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==1.13.1+cpu
 setuptools>=41.4.0
diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt
index 03dcf470f7..a32c6f6b1d 100644
--- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt
+++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt
@@ -8,3 +8,4 @@ wget
 pytorch-lightning==1.6.0
 deepspeed==0.3.15
 fairscale==0.4.6
+parameterized>=0.8.1
diff --git a/tools/ci_build/github/linux/docker/scripts/training/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/requirements.txt
deleted file mode 100644
index 331607c63c..0000000000
--- a/tools/ci_build/github/linux/docker/scripts/training/requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
---pre
--f https://download.pytorch.org/whl/torch_stable.html
-# transformers requires scikit-learn
-scikit-learn
-numpy==1.21.6
-transformers==v2.10.0
-torch==1.10.0+cu113
-torchvision==0.11.1+cu113
-torchtext==0.11.0
-tensorboard>=2.2.0,<2.5.0
-h5py
-setuptools<=59.5.0
-parameterized>=0.8.1