From 23a5d0a3052b11d8780c4ef646f0de0c2a542f84 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Sat, 6 Apr 2024 08:03:42 +0800 Subject: [PATCH] Extend time out in Windows GPU packaging jobs (#20207) ### Description Extend Windows GPU Packaging job building time out to 6 hours, and test stage to 3 hours. ### Motivation and Context There're still a few timeout issues after refactoring. The probability is about 20% in https://dev.azure.com/aiinfra/Lotus/_build?definitionId=84. I found the building could be finished in 4 hours if it becomes slow, https://dev.azure.com/aiinfra/Lotus/_build/results?buildId=434340&view=logs&j=0c6ee496-b38e-55a9-3699-12934156e90f, although in most cases, it only take about 30 minutes. Not like before, the building couldn't be completed. So, In this PR, I extend the timeout to 6 hours. And one interesting thing, if one windows GPU job becomes slow, all other windows GPU jobs in the same run become slow too. So I doubt it has something with the ADO or virtualization. That is, it's not completely random. https://dev.azure.com/aiinfra/Lotus/_build?definitionId=841 --- .../ci_build/github/azure-pipelines/templates/py-win-gpu.yml | 2 +- tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index b3a1650fbd..ab60446866 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -43,7 +43,7 @@ stages: dependsOn: [] jobs: - job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build - timeoutInMinutes: 120 + timeoutInMinutes: 360 workspace: clean: all pool: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index a6351e8c4f..f2a29ef8a4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -99,7 +99,7 @@ stages: ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: timeoutInMinutes: 1200 ${{ else }}: - timeoutInMinutes: 300 + timeoutInMinutes: 360 steps: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 @@ -313,7 +313,7 @@ stages: workspace: clean: all pool: ${{ parameters.ort_build_pool_name }} - timeoutInMinutes: 120 + timeoutInMinutes: 180 steps: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories'