From 96c0dbbe97b054742f9242b9e4a61d79693c849b Mon Sep 17 00:00:00 2001 From: Laith Sakka Date: Fri, 17 Jan 2025 07:57:40 +0000 Subject: [PATCH] Enhance running pr time benchmarks locally experience. (#144838) Summary: title Test Plan: NA Differential Revision: D68195894 Pull Request resolved: https://github.com/pytorch/pytorch/pull/144838 Approved by: https://github.com/huydhn --- .../pr_time_benchmarks/benchmark_runner.sh | 5 +- .../{ => benchmarks}/benchmark_base.py | 88 ++++++++++--------- .../pr_time_benchmarks/check_results.py | 7 ++ 3 files changed, 56 insertions(+), 44 deletions(-) rename benchmarks/dynamo/pr_time_benchmarks/{ => benchmarks}/benchmark_base.py (70%) diff --git a/benchmarks/dynamo/pr_time_benchmarks/benchmark_runner.sh b/benchmarks/dynamo/pr_time_benchmarks/benchmark_runner.sh index a5cf0417335..7a854a7020f 100644 --- a/benchmarks/dynamo/pr_time_benchmarks/benchmark_runner.sh +++ b/benchmarks/dynamo/pr_time_benchmarks/benchmark_runner.sh @@ -23,8 +23,9 @@ start=`date +%s` for file in $python_programs_dir/*.py do - # Execute the Python program and append the output to the output file - python $file $output_file + if [ "$file" != "$python_programs_dir/benchmark_base.py" ]; then + python $file $output_file + fi done end=`date +%s` diff --git a/benchmarks/dynamo/pr_time_benchmarks/benchmark_base.py b/benchmarks/dynamo/pr_time_benchmarks/benchmarks/benchmark_base.py similarity index 70% rename from benchmarks/dynamo/pr_time_benchmarks/benchmark_base.py rename to benchmarks/dynamo/pr_time_benchmarks/benchmarks/benchmark_base.py index fa41a5e49d8..7568fa701bb 100644 --- a/benchmarks/dynamo/pr_time_benchmarks/benchmark_base.py +++ b/benchmarks/dynamo/pr_time_benchmarks/benchmarks/benchmark_base.py @@ -4,55 +4,57 @@ import json import os from abc import ABC, abstractmethod -from fbscribelogger import make_scribe_logger - import torch._C._instruction_counter as i_counter import torch._dynamo.config as config from torch._dynamo.utils import CompileTimeInstructionCounter -scribe_log_torch_benchmark_compile_time = make_scribe_logger( - "TorchBenchmarkCompileTime", - """ -struct TorchBenchmarkCompileTimeLogEntry { +log_to_scuba = os.getenv("CI", "false") == "true" +if log_to_scuba: + from fbscribelogger import make_scribe_logger - # The commit SHA that triggered the workflow, e.g., 02a6b1d30f338206a71d0b75bfa09d85fac0028a. Derived from GITHUB_SHA. - 4: optional string commit_sha; + scribe_log_torch_benchmark_compile_time = make_scribe_logger( + "TorchBenchmarkCompileTime", + """ + struct TorchBenchmarkCompileTimeLogEntry { - # The unit timestamp in second for the Scuba Time Column override - 6: optional i64 time; - 7: optional i64 instruction_count; # Instruction count of compilation step - 8: optional string name; # Benchmark name + # The commit SHA that triggered the workflow, e.g., 02a6b1d30f338206a71d0b75bfa09d85fac0028a. Derived from GITHUB_SHA. + 4: optional string commit_sha; - # Commit date (not author date) of the commit in commit_sha as timestamp, e.g., 1724208105. Increasing if merge bot is used, though not monotonic; duplicates occur when stack is landed. - 16: optional i64 commit_date; + # The unit timestamp in second for the Scuba Time Column override + 6: optional i64 time; + 7: optional i64 instruction_count; # Instruction count of compilation step + 8: optional string name; # Benchmark name - # A unique number for each workflow run within a repository, e.g., 19471190684. Derived from GITHUB_RUN_ID. - 17: optional string github_run_id; + # Commit date (not author date) of the commit in commit_sha as timestamp, e.g., 1724208105. Increasing if merge bot is used, though not monotonic; duplicates occur when stack is landed. + 16: optional i64 commit_date; - # A unique number for each attempt of a particular workflow run in a repository, e.g., 1. Derived from GITHUB_RUN_ATTEMPT. - 18: optional string github_run_attempt; + # A unique number for each workflow run within a repository, e.g., 19471190684. Derived from GITHUB_RUN_ID. + 17: optional string github_run_id; - # Indicates if branch protections or rulesets are configured for the ref that triggered the workflow run. Derived from GITHUB_REF_PROTECTED. - 20: optional bool github_ref_protected; + # A unique number for each attempt of a particular workflow run in a repository, e.g., 1. Derived from GITHUB_RUN_ATTEMPT. + 18: optional string github_run_attempt; - # The fully-formed ref of the branch or tag that triggered the workflow run, e.g., refs/pull/133891/merge or refs/heads/main. Derived from GITHUB_REF. - 21: optional string github_ref; + # Indicates if branch protections or rulesets are configured for the ref that triggered the workflow run. Derived from GITHUB_REF_PROTECTED. + 20: optional bool github_ref_protected; - # The weight of the record according to current sampling rate - 25: optional i64 weight; + # The fully-formed ref of the branch or tag that triggered the workflow run, e.g., refs/pull/133891/merge or refs/heads/main. Derived from GITHUB_REF. + 21: optional string github_ref; - # The name of the current job. Derived from JOB_NAME, e.g., linux-jammy-py3.8-gcc11 / test (default, 3, 4, linux.2xlarge). - 26: optional string github_job; + # The weight of the record according to current sampling rate + 25: optional i64 weight; - # The GitHub user who triggered the job. Derived from GITHUB_TRIGGERING_ACTOR. - 27: optional string github_triggering_actor; + # The name of the current job. Derived from JOB_NAME, e.g., linux-jammy-py3.8-gcc11 / test (default, 3, 4, linux.2xlarge). + 26: optional string github_job; - # A unique number for each run of a particular workflow in a repository, e.g., 238742. Derived from GITHUB_RUN_NUMBER. - 28: optional string github_run_number_str; -} -""", # noqa: B950 -) + # The GitHub user who triggered the job. Derived from GITHUB_TRIGGERING_ACTOR. + 27: optional string github_triggering_actor; + + # A unique number for each run of a particular workflow in a repository, e.g., 238742. Derived from GITHUB_RUN_NUMBER. + 28: optional string github_run_number_str; + } + """, # noqa: B950 + ) class BenchmarkBase(ABC): @@ -237,10 +239,11 @@ class BenchmarkBase(ABC): if self._enable_instruction_count: r = self._count_instructions() self.results.append((self.name(), "instruction_count", r)) - scribe_log_torch_benchmark_compile_time( - name=self.name(), - instruction_count=r, - ) + if log_to_scuba: + scribe_log_torch_benchmark_compile_time( + name=self.name(), + instruction_count=r, + ) if self._enable_compile_time_instruction_count: r = self._count_compile_time_instructions() @@ -251,9 +254,10 @@ class BenchmarkBase(ABC): r, ) ) - # TODO add a new field compile_time_instruction_count to the logger. - scribe_log_torch_benchmark_compile_time( - name=self.name(), - instruction_count=r, - ) + if log_to_scuba: + # TODO add a new field compile_time_instruction_count to the logger. + scribe_log_torch_benchmark_compile_time( + name=self.name(), + instruction_count=r, + ) return self diff --git a/benchmarks/dynamo/pr_time_benchmarks/check_results.py b/benchmarks/dynamo/pr_time_benchmarks/check_results.py index 3c836c1b790..249d28ae502 100644 --- a/benchmarks/dynamo/pr_time_benchmarks/check_results.py +++ b/benchmarks/dynamo/pr_time_benchmarks/check_results.py @@ -219,6 +219,13 @@ def main(): f"There was some failures you can use the new reference expected result stored at path:" f"{reference_expected_results_path} and printed above\n" ) + print( + "To reproduce locally follow the following instructions, note that absolute instructions count are going " + "to be different than on the CI, hence you might want to run locally with and without your change:\n" + "cd benchmarks/dynamo/pr_time_benchmarks/ \n" + "python benchmarks/BENCHMARK.py result.csv \n" + "note that BENCHMARK.py is the name of the file containing the failing benchmark." + ) sys.exit(1) else: print("All benchmarks passed")