From 074f2bb5cebf84bf10e3c4daa76d7217905282dd Mon Sep 17 00:00:00 2001 From: chuanqiw Date: Tue, 6 Feb 2024 02:06:54 +0000 Subject: [PATCH] Fix dynamo benchmark runner for torchbench skip sets (#118615) Fix dynamo benchmark runner for torchbench skip sets, which introduced by PR #118032 This runner.py script is still used in the [Inductor CPU Performance Dashboard](https://github.com/pytorch/pytorch/issues/93531) regular test Pull Request resolved: https://github.com/pytorch/pytorch/pull/118615 Approved by: https://github.com/jgong5, https://github.com/ysiraichi, https://github.com/ezyang --- benchmarks/dynamo/runner.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/benchmarks/dynamo/runner.py b/benchmarks/dynamo/runner.py index 504e354a58b..37b46a430ea 100755 --- a/benchmarks/dynamo/runner.py +++ b/benchmarks/dynamo/runner.py @@ -378,10 +378,17 @@ def get_skip_tests(suite, is_training: bool): module = importlib.import_module(suite) os.chdir(original_dir) - if hasattr(module, "SKIP"): - skip_tests.update(module.SKIP) - if is_training and hasattr(module, "SKIP_TRAIN"): - skip_tests.update(module.SKIP_TRAIN) + if suite == "torchbench": + skip_tests.update(module.TorchBenchmarkRunner().skip_models) + if is_training: + skip_tests.update( + module.TorchBenchmarkRunner().skip_not_suitable_for_training_models + ) + else: + if hasattr(module, "SKIP"): + skip_tests.update(module.SKIP) + if is_training and hasattr(module, "SKIP_TRAIN"): + skip_tests.update(module.SKIP_TRAIN) skip_tests = (f"-x {name}" for name in skip_tests) skip_str = " ".join(skip_tests)