mirror of
https://github.com/saymrwulf/pytorch.git
synced 2026-05-14 20:57:59 +00:00
fb0f285638
Fixes https://github.com/pytorch/pytorch/issues/75927. Had to fix some bugs and add some ignores. To check if clean: ``` lintrunner --paths-cmd='git grep -Il .' --take MYPY,MYPYSTRICT ``` Pull Request resolved: https://github.com/pytorch/pytorch/pull/76753 Approved by: https://github.com/malfet
683 lines
22 KiB
Python
683 lines
22 KiB
Python
# -*- coding: utf-8 -*-
|
|
import unittest
|
|
from typing import Dict, List
|
|
|
|
from tools.stats import print_test_stats
|
|
from tools.stats.s3_stat_parser import (
|
|
Commit,
|
|
Report,
|
|
ReportMetaMeta,
|
|
Status,
|
|
Version1Case,
|
|
Version1Report,
|
|
Version2Case,
|
|
Version2Report,
|
|
)
|
|
|
|
|
|
def fakehash(char: str) -> str:
|
|
return char * 40
|
|
|
|
|
|
def dummy_meta_meta() -> ReportMetaMeta:
|
|
return {
|
|
"build_pr": "",
|
|
"build_tag": "",
|
|
"build_sha1": "",
|
|
"build_base_commit": "",
|
|
"build_branch": "",
|
|
"build_job": "",
|
|
"build_workflow_id": "",
|
|
"build_start_time_epoch": "",
|
|
}
|
|
|
|
|
|
def makecase(
|
|
name: str,
|
|
seconds: float,
|
|
*,
|
|
errored: bool = False,
|
|
failed: bool = False,
|
|
skipped: bool = False,
|
|
) -> Version1Case:
|
|
return {
|
|
"name": name,
|
|
"seconds": seconds,
|
|
"errored": errored,
|
|
"failed": failed,
|
|
"skipped": skipped,
|
|
}
|
|
|
|
|
|
def make_report_v1(tests: Dict[str, List[Version1Case]]) -> Version1Report:
|
|
suites = {
|
|
suite_name: {
|
|
"total_seconds": sum(case["seconds"] for case in cases),
|
|
"cases": cases,
|
|
}
|
|
for suite_name, cases in tests.items()
|
|
}
|
|
return {
|
|
**dummy_meta_meta(), # type: ignore[misc]
|
|
"total_seconds": sum(s["total_seconds"] for s in suites.values()),
|
|
"suites": suites,
|
|
}
|
|
|
|
|
|
def make_case_v2(seconds: float, status: Status = None) -> Version2Case:
|
|
return {
|
|
"seconds": seconds,
|
|
"status": status,
|
|
}
|
|
|
|
|
|
def make_report_v2(
|
|
tests: Dict[str, Dict[str, Dict[str, Version2Case]]]
|
|
) -> Version2Report:
|
|
files = {}
|
|
for file_name, file_suites in tests.items():
|
|
suites = {
|
|
suite_name: {
|
|
"total_seconds": sum(case["seconds"] for case in cases.values()),
|
|
"cases": cases,
|
|
}
|
|
for suite_name, cases in file_suites.items()
|
|
}
|
|
files[file_name] = {
|
|
"suites": suites,
|
|
"total_seconds": sum(suite["total_seconds"] for suite in suites.values()), # type: ignore[type-var]
|
|
}
|
|
return {
|
|
**dummy_meta_meta(), # type: ignore[misc]
|
|
"format_version": 2,
|
|
"total_seconds": sum(s["total_seconds"] for s in files.values()),
|
|
"files": files,
|
|
}
|
|
|
|
|
|
maxDiff = None
|
|
|
|
|
|
class TestPrintTestStats(unittest.TestCase):
|
|
version1_report: Version1Report = make_report_v1(
|
|
{
|
|
# input ordering of the suites is ignored
|
|
"Grault": [
|
|
# not printed: status same and time similar
|
|
makecase("test_grault0", 4.78, failed=True),
|
|
# status same, but time increased a lot
|
|
makecase("test_grault2", 1.473, errored=True),
|
|
],
|
|
# individual tests times changed, not overall suite
|
|
"Qux": [
|
|
# input ordering of the test cases is ignored
|
|
makecase("test_qux1", 0.001, skipped=True),
|
|
makecase("test_qux6", 0.002, skipped=True),
|
|
# time in bounds, but status changed
|
|
makecase("test_qux4", 7.158, failed=True),
|
|
# not printed because it's the same as before
|
|
makecase("test_qux7", 0.003, skipped=True),
|
|
makecase("test_qux5", 11.968),
|
|
makecase("test_qux3", 23.496),
|
|
],
|
|
# new test suite
|
|
"Bar": [
|
|
makecase("test_bar2", 3.742, failed=True),
|
|
makecase("test_bar1", 50.447),
|
|
],
|
|
# overall suite time changed but no individual tests
|
|
"Norf": [
|
|
makecase("test_norf1", 3),
|
|
makecase("test_norf2", 3),
|
|
makecase("test_norf3", 3),
|
|
makecase("test_norf4", 3),
|
|
],
|
|
# suite doesn't show up if it doesn't change enough
|
|
"Foo": [
|
|
makecase("test_foo1", 42),
|
|
makecase("test_foo2", 56),
|
|
],
|
|
}
|
|
)
|
|
|
|
version2_report: Version2Report = make_report_v2(
|
|
{
|
|
"test_a": {
|
|
"Grault": {
|
|
"test_grault0": make_case_v2(4.78, "failed"),
|
|
"test_grault2": make_case_v2(1.473, "errored"),
|
|
},
|
|
"Qux": {
|
|
"test_qux1": make_case_v2(0.001, "skipped"),
|
|
"test_qux6": make_case_v2(0.002, "skipped"),
|
|
"test_qux4": make_case_v2(7.158, "failed"),
|
|
"test_qux7": make_case_v2(0.003, "skipped"),
|
|
"test_qux8": make_case_v2(11.968),
|
|
"test_qux3": make_case_v2(23.496),
|
|
},
|
|
},
|
|
"test_b": {
|
|
"Bar": {
|
|
"test_bar2": make_case_v2(3.742, "failed"),
|
|
"test_bar1": make_case_v2(50.447),
|
|
},
|
|
# overall suite time changed but no individual tests
|
|
"Norf": {
|
|
"test_norf1": make_case_v2(3),
|
|
"test_norf2": make_case_v2(3),
|
|
"test_norf3": make_case_v2(3),
|
|
"test_norf4": make_case_v2(3),
|
|
},
|
|
},
|
|
"test_c": {
|
|
"Foo": {
|
|
"test_foo1": make_case_v2(42),
|
|
"test_foo2": make_case_v2(56),
|
|
},
|
|
},
|
|
}
|
|
)
|
|
|
|
def test_simplify(self) -> None:
|
|
self.assertEqual(
|
|
{
|
|
"": {
|
|
"Bar": {
|
|
"test_bar1": {"seconds": 50.447, "status": None},
|
|
"test_bar2": {"seconds": 3.742, "status": "failed"},
|
|
},
|
|
"Foo": {
|
|
"test_foo1": {"seconds": 42, "status": None},
|
|
"test_foo2": {"seconds": 56, "status": None},
|
|
},
|
|
"Grault": {
|
|
"test_grault0": {"seconds": 4.78, "status": "failed"},
|
|
"test_grault2": {"seconds": 1.473, "status": "errored"},
|
|
},
|
|
"Norf": {
|
|
"test_norf1": {"seconds": 3, "status": None},
|
|
"test_norf3": {"seconds": 3, "status": None},
|
|
"test_norf2": {"seconds": 3, "status": None},
|
|
"test_norf4": {"seconds": 3, "status": None},
|
|
},
|
|
"Qux": {
|
|
"test_qux1": {"seconds": 0.001, "status": "skipped"},
|
|
"test_qux3": {"seconds": 23.496, "status": None},
|
|
"test_qux4": {"seconds": 7.158, "status": "failed"},
|
|
"test_qux5": {"seconds": 11.968, "status": None},
|
|
"test_qux6": {"seconds": 0.002, "status": "skipped"},
|
|
"test_qux7": {"seconds": 0.003, "status": "skipped"},
|
|
},
|
|
},
|
|
},
|
|
print_test_stats.simplify(self.version1_report),
|
|
)
|
|
|
|
self.assertEqual(
|
|
{
|
|
"test_a": {
|
|
"Grault": {
|
|
"test_grault0": {"seconds": 4.78, "status": "failed"},
|
|
"test_grault2": {"seconds": 1.473, "status": "errored"},
|
|
},
|
|
"Qux": {
|
|
"test_qux1": {"seconds": 0.001, "status": "skipped"},
|
|
"test_qux3": {"seconds": 23.496, "status": None},
|
|
"test_qux4": {"seconds": 7.158, "status": "failed"},
|
|
"test_qux6": {"seconds": 0.002, "status": "skipped"},
|
|
"test_qux7": {"seconds": 0.003, "status": "skipped"},
|
|
"test_qux8": {"seconds": 11.968, "status": None},
|
|
},
|
|
},
|
|
"test_b": {
|
|
"Bar": {
|
|
"test_bar1": {"seconds": 50.447, "status": None},
|
|
"test_bar2": {"seconds": 3.742, "status": "failed"},
|
|
},
|
|
"Norf": {
|
|
"test_norf1": {"seconds": 3, "status": None},
|
|
"test_norf2": {"seconds": 3, "status": None},
|
|
"test_norf3": {"seconds": 3, "status": None},
|
|
"test_norf4": {"seconds": 3, "status": None},
|
|
},
|
|
},
|
|
"test_c": {
|
|
"Foo": {
|
|
"test_foo1": {"seconds": 42, "status": None},
|
|
"test_foo2": {"seconds": 56, "status": None},
|
|
},
|
|
},
|
|
},
|
|
print_test_stats.simplify(self.version2_report),
|
|
)
|
|
|
|
def test_analysis(self) -> None:
|
|
head_report = self.version1_report
|
|
|
|
base_reports: Dict[Commit, List[Report]] = {
|
|
# bbbb has no reports, so base is cccc instead
|
|
fakehash("b"): [],
|
|
fakehash("c"): [
|
|
make_report_v1(
|
|
{
|
|
"Baz": [
|
|
makecase("test_baz2", 13.605),
|
|
# no recent suites have & skip this test
|
|
makecase("test_baz1", 0.004, skipped=True),
|
|
],
|
|
"Foo": [
|
|
makecase("test_foo1", 43),
|
|
# test added since dddd
|
|
makecase("test_foo2", 57),
|
|
],
|
|
"Grault": [
|
|
makecase("test_grault0", 4.88, failed=True),
|
|
makecase("test_grault1", 11.967, failed=True),
|
|
makecase("test_grault2", 0.395, errored=True),
|
|
makecase("test_grault3", 30.460),
|
|
],
|
|
"Norf": [
|
|
makecase("test_norf1", 2),
|
|
makecase("test_norf2", 2),
|
|
makecase("test_norf3", 2),
|
|
makecase("test_norf4", 2),
|
|
],
|
|
"Qux": [
|
|
makecase("test_qux3", 4.978, errored=True),
|
|
makecase("test_qux7", 0.002, skipped=True),
|
|
makecase("test_qux2", 5.618),
|
|
makecase("test_qux4", 7.766, errored=True),
|
|
makecase("test_qux6", 23.589, failed=True),
|
|
],
|
|
}
|
|
),
|
|
],
|
|
fakehash("d"): [
|
|
make_report_v1(
|
|
{
|
|
"Foo": [
|
|
makecase("test_foo1", 40),
|
|
# removed in cccc
|
|
makecase("test_foo3", 17),
|
|
],
|
|
"Baz": [
|
|
# not skipped, so not included in stdev
|
|
makecase("test_baz1", 3.14),
|
|
],
|
|
"Qux": [
|
|
makecase("test_qux7", 0.004, skipped=True),
|
|
makecase("test_qux2", 6.02),
|
|
makecase("test_qux4", 20.932),
|
|
],
|
|
"Norf": [
|
|
makecase("test_norf1", 3),
|
|
makecase("test_norf2", 3),
|
|
makecase("test_norf3", 3),
|
|
makecase("test_norf4", 3),
|
|
],
|
|
"Grault": [
|
|
makecase("test_grault0", 5, failed=True),
|
|
makecase("test_grault1", 14.325, failed=True),
|
|
makecase("test_grault2", 0.31, errored=True),
|
|
],
|
|
}
|
|
),
|
|
],
|
|
fakehash("e"): [],
|
|
fakehash("f"): [
|
|
make_report_v1(
|
|
{
|
|
"Foo": [
|
|
makecase("test_foo3", 24),
|
|
makecase("test_foo1", 43),
|
|
],
|
|
"Baz": [
|
|
makecase("test_baz2", 16.857),
|
|
],
|
|
"Qux": [
|
|
makecase("test_qux2", 6.422),
|
|
makecase("test_qux4", 6.382, errored=True),
|
|
],
|
|
"Norf": [
|
|
makecase("test_norf1", 0.9),
|
|
makecase("test_norf3", 0.9),
|
|
makecase("test_norf2", 0.9),
|
|
makecase("test_norf4", 0.9),
|
|
],
|
|
"Grault": [
|
|
makecase("test_grault0", 4.7, failed=True),
|
|
makecase("test_grault1", 13.146, failed=True),
|
|
makecase("test_grault2", 0.48, errored=True),
|
|
],
|
|
}
|
|
),
|
|
],
|
|
}
|
|
|
|
simpler_head = print_test_stats.simplify(head_report)
|
|
simpler_base = {}
|
|
for commit, reports in base_reports.items():
|
|
simpler_base[commit] = [print_test_stats.simplify(r) for r in reports]
|
|
analysis = print_test_stats.analyze(
|
|
head_report=simpler_head,
|
|
base_reports=simpler_base,
|
|
)
|
|
|
|
self.assertEqual(
|
|
"""\
|
|
|
|
- class Baz:
|
|
- # was 15.23s ± 2.30s
|
|
-
|
|
- def test_baz1: ...
|
|
- # was 0.004s (skipped)
|
|
-
|
|
- def test_baz2: ...
|
|
- # was 15.231s ± 2.300s
|
|
|
|
|
|
class Grault:
|
|
# was 48.86s ± 1.19s
|
|
# now 6.25s
|
|
|
|
- def test_grault1: ...
|
|
- # was 13.146s ± 1.179s (failed)
|
|
|
|
- def test_grault3: ...
|
|
- # was 30.460s
|
|
|
|
|
|
class Qux:
|
|
# was 41.66s ± 1.06s
|
|
# now 42.63s
|
|
|
|
- def test_qux2: ...
|
|
- # was 6.020s ± 0.402s
|
|
|
|
! def test_qux3: ...
|
|
! # was 4.978s (errored)
|
|
! # now 23.496s
|
|
|
|
! def test_qux4: ...
|
|
! # was 7.074s ± 0.979s (errored)
|
|
! # now 7.158s (failed)
|
|
|
|
! def test_qux6: ...
|
|
! # was 23.589s (failed)
|
|
! # now 0.002s (skipped)
|
|
|
|
+ def test_qux1: ...
|
|
+ # now 0.001s (skipped)
|
|
|
|
+ def test_qux5: ...
|
|
+ # now 11.968s
|
|
|
|
|
|
+ class Bar:
|
|
+ # now 54.19s
|
|
+
|
|
+ def test_bar1: ...
|
|
+ # now 50.447s
|
|
+
|
|
+ def test_bar2: ...
|
|
+ # now 3.742s (failed)
|
|
|
|
""",
|
|
print_test_stats.anomalies(analysis),
|
|
)
|
|
|
|
def test_graph(self) -> None:
|
|
# HEAD is on master
|
|
self.assertEqual(
|
|
"""\
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
* aaaaaaaaaa (HEAD) total time 502.99s
|
|
* bbbbbbbbbb (base) 1 report, total time 47.84s
|
|
* cccccccccc 1 report, total time 332.50s
|
|
* dddddddddd 0 reports
|
|
|
|
|
:
|
|
""",
|
|
print_test_stats.graph(
|
|
head_sha=fakehash("a"),
|
|
head_seconds=502.99,
|
|
base_seconds={
|
|
fakehash("b"): [47.84],
|
|
fakehash("c"): [332.50],
|
|
fakehash("d"): [],
|
|
},
|
|
on_master=True,
|
|
),
|
|
)
|
|
|
|
self.assertEqual(
|
|
"""\
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
| * aaaaaaaaaa (HEAD) total time 9988.77s
|
|
|/
|
|
* bbbbbbbbbb (base) 121 reports, total time 7654.32s ± 55.55s
|
|
* cccccccccc 20 reports, total time 5555.55s ± 253.19s
|
|
* dddddddddd 1 report, total time 1234.56s
|
|
|
|
|
:
|
|
""",
|
|
print_test_stats.graph(
|
|
head_sha=fakehash("a"),
|
|
head_seconds=9988.77,
|
|
base_seconds={
|
|
fakehash("b"): [7598.77] * 60 + [7654.32] + [7709.87] * 60,
|
|
fakehash("c"): [5308.77] * 10 + [5802.33] * 10,
|
|
fakehash("d"): [1234.56],
|
|
},
|
|
on_master=False,
|
|
),
|
|
)
|
|
|
|
self.assertEqual(
|
|
"""\
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
| * aaaaaaaaaa (HEAD) total time 25.52s
|
|
| |
|
|
| : (5 commits)
|
|
|/
|
|
* bbbbbbbbbb 0 reports
|
|
* cccccccccc 0 reports
|
|
* dddddddddd (base) 15 reports, total time 58.92s ± 25.82s
|
|
|
|
|
:
|
|
""",
|
|
print_test_stats.graph(
|
|
head_sha=fakehash("a"),
|
|
head_seconds=25.52,
|
|
base_seconds={
|
|
fakehash("b"): [],
|
|
fakehash("c"): [],
|
|
fakehash("d"): [52.25] * 14 + [152.26],
|
|
},
|
|
on_master=False,
|
|
ancestry_path=5,
|
|
),
|
|
)
|
|
|
|
self.assertEqual(
|
|
"""\
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
| * aaaaaaaaaa (HEAD) total time 0.08s
|
|
|/|
|
|
| : (1 commit)
|
|
|
|
|
* bbbbbbbbbb 0 reports
|
|
* cccccccccc (base) 1 report, total time 0.09s
|
|
* dddddddddd 3 reports, total time 0.10s ± 0.05s
|
|
|
|
|
:
|
|
""",
|
|
print_test_stats.graph(
|
|
head_sha=fakehash("a"),
|
|
head_seconds=0.08,
|
|
base_seconds={
|
|
fakehash("b"): [],
|
|
fakehash("c"): [0.09],
|
|
fakehash("d"): [0.05, 0.10, 0.15],
|
|
},
|
|
on_master=False,
|
|
other_ancestors=1,
|
|
),
|
|
)
|
|
|
|
self.assertEqual(
|
|
"""\
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
| * aaaaaaaaaa (HEAD) total time 5.98s
|
|
| |
|
|
| : (1 commit)
|
|
|/|
|
|
| : (7 commits)
|
|
|
|
|
* bbbbbbbbbb (base) 2 reports, total time 6.02s ± 1.71s
|
|
* cccccccccc 0 reports
|
|
* dddddddddd 10 reports, total time 5.84s ± 0.92s
|
|
|
|
|
:
|
|
""",
|
|
print_test_stats.graph(
|
|
head_sha=fakehash("a"),
|
|
head_seconds=5.98,
|
|
base_seconds={
|
|
fakehash("b"): [4.81, 7.23],
|
|
fakehash("c"): [],
|
|
fakehash("d"): [4.97] * 5 + [6.71] * 5,
|
|
},
|
|
on_master=False,
|
|
ancestry_path=1,
|
|
other_ancestors=7,
|
|
),
|
|
)
|
|
|
|
def test_regression_info(self) -> None:
|
|
self.assertEqual(
|
|
"""\
|
|
----- Historic stats comparison result ------
|
|
|
|
job: foo_job
|
|
commit: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
|
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
| * aaaaaaaaaa (HEAD) total time 3.02s
|
|
|/
|
|
* bbbbbbbbbb (base) 1 report, total time 41.00s
|
|
* cccccccccc 1 report, total time 43.00s
|
|
|
|
|
:
|
|
|
|
Removed (across 1 suite) 1 test, totaling - 1.00s
|
|
Modified (across 1 suite) 1 test, totaling - 41.48s ± 2.12s
|
|
Added (across 1 suite) 1 test, totaling + 3.00s
|
|
""",
|
|
print_test_stats.regression_info(
|
|
head_sha=fakehash("a"),
|
|
head_report=make_report_v1(
|
|
{
|
|
"Foo": [
|
|
makecase("test_foo", 0.02, skipped=True),
|
|
makecase("test_baz", 3),
|
|
]
|
|
}
|
|
),
|
|
base_reports={
|
|
fakehash("b"): [
|
|
make_report_v1(
|
|
{
|
|
"Foo": [
|
|
makecase("test_foo", 40),
|
|
makecase("test_bar", 1),
|
|
],
|
|
}
|
|
),
|
|
],
|
|
fakehash("c"): [
|
|
make_report_v1(
|
|
{
|
|
"Foo": [
|
|
makecase("test_foo", 43),
|
|
],
|
|
}
|
|
),
|
|
],
|
|
},
|
|
job_name="foo_job",
|
|
on_master=False,
|
|
ancestry_path=0,
|
|
other_ancestors=0,
|
|
),
|
|
)
|
|
|
|
def test_regression_info_new_job(self) -> None:
|
|
self.assertEqual(
|
|
"""\
|
|
----- Historic stats comparison result ------
|
|
|
|
job: foo_job
|
|
commit: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
|
|
|
Commit graph (base is most recent master ancestor with at least one S3 report):
|
|
|
|
: (master)
|
|
|
|
|
| * aaaaaaaaaa (HEAD) total time 3.02s
|
|
| |
|
|
| : (3 commits)
|
|
|/|
|
|
| : (2 commits)
|
|
|
|
|
* bbbbbbbbbb 0 reports
|
|
* cccccccccc 0 reports
|
|
|
|
|
:
|
|
|
|
Removed (across 0 suites) 0 tests, totaling 0.00s
|
|
Modified (across 0 suites) 0 tests, totaling 0.00s
|
|
Added (across 1 suite) 2 tests, totaling + 3.02s
|
|
""",
|
|
print_test_stats.regression_info(
|
|
head_sha=fakehash("a"),
|
|
head_report=make_report_v1(
|
|
{
|
|
"Foo": [
|
|
makecase("test_foo", 0.02, skipped=True),
|
|
makecase("test_baz", 3),
|
|
]
|
|
}
|
|
),
|
|
base_reports={
|
|
fakehash("b"): [],
|
|
fakehash("c"): [],
|
|
},
|
|
job_name="foo_job",
|
|
on_master=False,
|
|
ancestry_path=3,
|
|
other_ancestors=2,
|
|
),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|