mirror of
https://github.com/saymrwulf/pytorch.git
synced 2026-05-14 20:57:59 +00:00
06a0cfc0ea
This PR uses pytest to run test_ops, test_ops_gradients, and test_ops_jit in parallel in non linux cuda environments to decrease TTS. I am excluding linux cuda because running in parallel results in errors due to running out of memory Notes: * update hypothesis version for compatability with pytest * use rerun-failures to rerun tests (similar to flaky tests, although these test files generally don't have flaky tests) * reruns are denoted by a rerun tag in the xml. Failed reruns also have the failure tag. Successes (meaning that the test is flaky) do not have the failure tag. * see https://docs.google.com/spreadsheets/d/1aO0Rbg3y3ch7ghipt63PG2KNEUppl9a5b18Hmv2CZ4E/edit#gid=602543594 for info on speedup (or slowdown in the case of slow tests) * expecting windows tests to decrease by 60 minutes total * slow test infra is expected to stay the same - verified by running pytest and unittest on the same job and check the number of skipped/run tests * test reports to s3 changed - add entirely new table to keep track of invoking_file times Pull Request resolved: https://github.com/pytorch/pytorch/pull/79898 Approved by: https://github.com/malfet, https://github.com/janeyx99
146 lines
5 KiB
Python
146 lines
5 KiB
Python
from _pytest.junitxml import LogXML, _NodeReporter, bin_xml_escape
|
|
from _pytest.terminal import _get_raw_skip_reason
|
|
from _pytest.stash import StashKey
|
|
from _pytest.reports import TestReport
|
|
from _pytest.config.argparsing import Parser
|
|
from _pytest.config import filename_arg
|
|
from _pytest.config import Config
|
|
from _pytest._code.code import ReprFileLocation
|
|
from typing import Union
|
|
from typing import Optional
|
|
import xml.etree.ElementTree as ET
|
|
import functools
|
|
|
|
# a lot of this file is copied from _pytest.junitxml and modified to get rerun info
|
|
|
|
xml_key = StashKey["LogXMLReruns"]()
|
|
|
|
|
|
def pytest_addoption(parser: Parser) -> None:
|
|
group = parser.getgroup("terminal reporting")
|
|
group.addoption(
|
|
"--junit-xml-reruns",
|
|
action="store",
|
|
dest="xmlpath_reruns",
|
|
metavar="path",
|
|
type=functools.partial(filename_arg, optname="--junit-xml-reruns"),
|
|
default=None,
|
|
help="create junit-xml style report file at given path.",
|
|
)
|
|
group.addoption(
|
|
"--junit-prefix-reruns",
|
|
action="store",
|
|
metavar="str",
|
|
default=None,
|
|
help="prepend prefix to classnames in junit-xml output",
|
|
)
|
|
parser.addini(
|
|
"junit_suite_name_reruns", "Test suite name for JUnit report", default="pytest"
|
|
)
|
|
parser.addini(
|
|
"junit_logging_reruns",
|
|
"Write captured log messages to JUnit report: "
|
|
"one of no|log|system-out|system-err|out-err|all",
|
|
default="no",
|
|
)
|
|
parser.addini(
|
|
"junit_log_passing_tests_reruns",
|
|
"Capture log information for passing tests to JUnit report: ",
|
|
type="bool",
|
|
default=True,
|
|
)
|
|
parser.addini(
|
|
"junit_duration_report_reruns",
|
|
"Duration time to report: one of total|call",
|
|
default="total",
|
|
)
|
|
parser.addini(
|
|
"junit_family_reruns",
|
|
"Emit XML for schema: one of legacy|xunit1|xunit2",
|
|
default="xunit2",
|
|
)
|
|
|
|
|
|
def pytest_configure(config: Config) -> None:
|
|
xmlpath = config.option.xmlpath_reruns
|
|
# Prevent opening xmllog on worker nodes (xdist).
|
|
if xmlpath and not hasattr(config, "workerinput"):
|
|
junit_family = config.getini("junit_family_reruns")
|
|
config.stash[xml_key] = LogXMLReruns(
|
|
xmlpath,
|
|
config.option.junitprefix,
|
|
config.getini("junit_suite_name_reruns"),
|
|
config.getini("junit_logging_reruns"),
|
|
config.getini("junit_duration_report_reruns"),
|
|
junit_family,
|
|
config.getini("junit_log_passing_tests_reruns"),
|
|
)
|
|
config.pluginmanager.register(config.stash[xml_key])
|
|
|
|
|
|
def pytest_unconfigure(config: Config) -> None:
|
|
xml = config.stash.get(xml_key, None)
|
|
if xml:
|
|
del config.stash[xml_key]
|
|
config.pluginmanager.unregister(xml)
|
|
|
|
|
|
class _NodeReporterReruns(_NodeReporter):
|
|
def _prepare_content(self, content: str, header: str) -> str:
|
|
return content
|
|
|
|
def _write_content(self, report: TestReport, content: str, jheader: str) -> None:
|
|
if content == "":
|
|
return
|
|
tag = ET.Element(jheader)
|
|
tag.text = bin_xml_escape(content)
|
|
self.append(tag)
|
|
|
|
|
|
class LogXMLReruns(LogXML):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
|
|
def append_rerun(self, reporter: _NodeReporter, report: TestReport) -> None:
|
|
if hasattr(report, "wasxfail"):
|
|
reporter._add_simple("skipped", "xfail-marked test passes unexpectedly")
|
|
else:
|
|
assert report.longrepr is not None
|
|
reprcrash: Optional[ReprFileLocation] = getattr(
|
|
report.longrepr, "reprcrash", None
|
|
)
|
|
if reprcrash is not None:
|
|
message = reprcrash.message
|
|
else:
|
|
message = str(report.longrepr)
|
|
message = bin_xml_escape(message)
|
|
reporter._add_simple("rerun", message, str(report.longrepr))
|
|
|
|
def pytest_runtest_logreport(self, report: TestReport) -> None:
|
|
super().pytest_runtest_logreport(report)
|
|
if report.outcome == "rerun":
|
|
reporter = self._opentestcase(report)
|
|
self.append_rerun(reporter, report)
|
|
if report.outcome == "skipped":
|
|
if isinstance(report.longrepr, tuple):
|
|
fspath, lineno, reason = report.longrepr
|
|
reason = f"{report.nodeid}: {_get_raw_skip_reason(report)}"
|
|
report.longrepr = (fspath, lineno, reason)
|
|
|
|
def node_reporter(self, report: Union[TestReport, str]) -> _NodeReporterReruns:
|
|
nodeid: Union[str, TestReport] = getattr(report, "nodeid", report)
|
|
# Local hack to handle xdist report order.
|
|
workernode = getattr(report, "node", None)
|
|
|
|
key = nodeid, workernode
|
|
|
|
if key in self.node_reporters:
|
|
# TODO: breaks for --dist=each
|
|
return self.node_reporters[key]
|
|
|
|
reporter = _NodeReporterReruns(nodeid, self)
|
|
|
|
self.node_reporters[key] = reporter
|
|
self.node_reporters_ordered.append(reporter)
|
|
|
|
return reporter
|