mirror of
https://github.com/saymrwulf/pytorch.git
synced 2026-05-15 21:00:47 +00:00
9470059766
Allow `viable/strict` promotion even if `periodic` or `docker-release-builds` jobs are failing **Why?** Those jobs only run occasionally and for all we know the current viable/strict commit may already include the errors that the above cron based workflows may have later detected. Blocking the viable/strict upgrade because of these scheduled jobs doesn't really offer any value, it just leads to people getting older PRs when they try to fork off of viable/strict without guaranteeing an improvement in test quality Though frankly, the current situation is worse than that. Assume the branch history looks like A -> B A is the current `viable/strict` commit B is a commit that failed some `periodic` test, so `viable/strict` wasn't upgraded to B Now lets say there's a commit C that gets merged. C neither contains a fix for the failing periodic build, nor does a scheduled periodic workflow run against C. The branch becomes A -> B -> C In the above scenario, today we will promote `viable/strict` to C since there was no failing workflow there!!! Even though it didn't actually fix what was broken with B! In short, avoiding the upgrade to B really doesn't make any sense today and we shouldn't do it. Pull Request resolved: https://github.com/pytorch/pytorch/pull/86827 Approved by: https://github.com/janeyx99
100 lines
4.8 KiB
Python
100 lines
4.8 KiB
Python
from unittest import TestCase, main, mock
|
|
from typing import Any, List, Dict
|
|
from fetch_latest_green_commit import isGreen, WorkflowCheck
|
|
|
|
workflowNames = [
|
|
"pull",
|
|
"trunk",
|
|
"Lint",
|
|
"linux-binary-libtorch-pre-cxx11",
|
|
"android-tests",
|
|
"windows-binary-wheel",
|
|
"periodic",
|
|
"docker-release-builds",
|
|
"nightly",
|
|
"pr-labels",
|
|
"Close stale pull requests",
|
|
"Update S3 HTML indices for download.pytorch.org",
|
|
"Create Release"
|
|
]
|
|
|
|
def set_workflow_job_status(workflow: List[Dict[str, Any]], name: str, status: str) -> List[Dict[str, Any]]:
|
|
for check in workflow:
|
|
if check['workflowName'] == name:
|
|
check['conclusion'] = status
|
|
return workflow
|
|
|
|
class TestChecks:
|
|
def make_test_checks(self) -> List[Dict[str, Any]]:
|
|
workflow_checks = []
|
|
for i in range(len(workflowNames)):
|
|
workflow_checks.append(WorkflowCheck(
|
|
workflowName=workflowNames[i],
|
|
name="test/job",
|
|
jobName="job",
|
|
conclusion="success",
|
|
)._asdict())
|
|
return workflow_checks
|
|
|
|
class TestPrintCommits(TestCase):
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value=TestChecks().make_test_checks())
|
|
def test_all_successful(self, mock_get_commit_results: Any) -> None:
|
|
"Test with workflows are successful"
|
|
workflow_checks = mock_get_commit_results()
|
|
self.assertTrue(isGreen("sha", workflow_checks)[0])
|
|
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value=TestChecks().make_test_checks())
|
|
def test_necessary_successful(self, mock_get_commit_results: Any) -> None:
|
|
"Test with necessary workflows are successful"
|
|
workflow_checks = mock_get_commit_results()
|
|
workflow_checks = set_workflow_job_status(workflow_checks, workflowNames[8], "failed")
|
|
workflow_checks = set_workflow_job_status(workflow_checks, workflowNames[9], "failed")
|
|
workflow_checks = set_workflow_job_status(workflow_checks, workflowNames[10], "failed")
|
|
workflow_checks = set_workflow_job_status(workflow_checks, workflowNames[11], "failed")
|
|
workflow_checks = set_workflow_job_status(workflow_checks, workflowNames[12], "failed")
|
|
self.assertTrue(isGreen("sha", workflow_checks)[0])
|
|
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value=TestChecks().make_test_checks())
|
|
def test_necessary_skipped(self, mock_get_commit_results: Any) -> None:
|
|
"Test with necessary job (ex: pull) skipped"
|
|
workflow_checks = mock_get_commit_results()
|
|
workflow_checks = set_workflow_job_status(workflow_checks, "pull", "skipped")
|
|
result = isGreen("sha", workflow_checks)
|
|
self.assertTrue(result[0])
|
|
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value=TestChecks().make_test_checks())
|
|
def test_skippable_skipped(self, mock_get_commit_results: Any) -> None:
|
|
"Test with skippable jobs (periodic and docker-release-builds skipped"
|
|
workflow_checks = mock_get_commit_results()
|
|
workflow_checks = set_workflow_job_status(workflow_checks, "periodic", "skipped")
|
|
workflow_checks = set_workflow_job_status(workflow_checks, "docker-release-builds", "skipped")
|
|
self.assertTrue(isGreen("sha", workflow_checks))
|
|
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value=TestChecks().make_test_checks())
|
|
def test_necessary_failed(self, mock_get_commit_results: Any) -> None:
|
|
"Test with necessary job (ex: Lint) failed"
|
|
workflow_checks = mock_get_commit_results()
|
|
workflow_checks = set_workflow_job_status(workflow_checks, "Lint", "failed")
|
|
result = isGreen("sha", workflow_checks)
|
|
self.assertFalse(result[0])
|
|
self.assertEqual(result[1], "Lint checks were not successful")
|
|
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value=TestChecks().make_test_checks())
|
|
def test_skippable_failed(self, mock_get_commit_results: Any) -> None:
|
|
"Test with failing skippable jobs (ex: docker-release-builds) should pass"
|
|
workflow_checks = mock_get_commit_results()
|
|
workflow_checks = set_workflow_job_status(workflow_checks, "periodic", "skipped")
|
|
workflow_checks = set_workflow_job_status(workflow_checks, "docker-release-builds", "failed")
|
|
result = isGreen("sha", workflow_checks)
|
|
self.assertTrue(result[0])
|
|
|
|
@mock.patch('fetch_latest_green_commit.get_commit_results', return_value={})
|
|
def test_no_workflows(self, mock_get_commit_results: Any) -> None:
|
|
"Test with missing workflows"
|
|
workflow_checks = mock_get_commit_results()
|
|
result = isGreen("sha", workflow_checks)
|
|
self.assertFalse(result[0])
|
|
self.assertEqual(result[1], "missing required workflows: pull, trunk, lint, linux-binary, windows-binary")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|