saymrwulf/pytorch
1
0
Fork 0
mirror of https://github.com/saymrwulf/pytorch.git synced 2026-05-14 20:57:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enhance running pr time benchmarks locally experience. (#144838)

Browse source 
Summary: title

Test Plan: NA

Differential Revision: D68195894

Pull Request resolved: https://github.com/pytorch/pytorch/pull/144838
Approved by: https://github.com/huydhn
This commit is contained in:
Laith Sakka 2025-01-17 07:57:40 +00:00 committed by PyTorch MergeBot
parent 465a1cfe2e
commit 96c0dbbe97
3 changed files with 56 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
benchmarks/dynamo/pr_time_benchmarks/benchmark_runner.sh
View file

@ -23,8 +23,9 @@ start=`date +%s`
for file in $python_programs_dir/*.py
do
# Execute the Python program and append the output to the output file
python $file $output_file
if [ "$file" != "$python_programs_dir/benchmark_base.py" ]; then
python $file $output_file
fi
done
end=`date +%s`

88
benchmarks/dynamo/pr_time_benchmarks/benchmark_base.py → benchmarks/dynamo/pr_time_benchmarks/benchmarks/benchmark_base.py
View file

@ -4,55 +4,57 @@ import json
import os
from abc import ABC, abstractmethod
from fbscribelogger import make_scribe_logger
import torch._C._instruction_counter as i_counter
import torch._dynamo.config as config
from torch._dynamo.utils import CompileTimeInstructionCounter
scribe_log_torch_benchmark_compile_time = make_scribe_logger(
"TorchBenchmarkCompileTime",
"""
struct TorchBenchmarkCompileTimeLogEntry {
log_to_scuba = os.getenv("CI", "false") == "true"
if log_to_scuba:
from fbscribelogger import make_scribe_logger
# The commit SHA that triggered the workflow, e.g., 02a6b1d30f338206a71d0b75bfa09d85fac0028a. Derived from GITHUB_SHA.
4: optional string commit_sha;
scribe_log_torch_benchmark_compile_time = make_scribe_logger(
"TorchBenchmarkCompileTime",
"""
struct TorchBenchmarkCompileTimeLogEntry {
# The unit timestamp in second for the Scuba Time Column override
6: optional i64 time;
7: optional i64 instruction_count; # Instruction count of compilation step
8: optional string name; # Benchmark name
# The commit SHA that triggered the workflow, e.g., 02a6b1d30f338206a71d0b75bfa09d85fac0028a. Derived from GITHUB_SHA.
4: optional string commit_sha;
# Commit date (not author date) of the commit in commit_sha as timestamp, e.g., 1724208105. Increasing if merge bot is used, though not monotonic; duplicates occur when stack is landed.
16: optional i64 commit_date;
# The unit timestamp in second for the Scuba Time Column override
6: optional i64 time;
7: optional i64 instruction_count; # Instruction count of compilation step
8: optional string name; # Benchmark name
# A unique number for each workflow run within a repository, e.g., 19471190684. Derived from GITHUB_RUN_ID.
17: optional string github_run_id;
# Commit date (not author date) of the commit in commit_sha as timestamp, e.g., 1724208105. Increasing if merge bot is used, though not monotonic; duplicates occur when stack is landed.
16: optional i64 commit_date;
# A unique number for each attempt of a particular workflow run in a repository, e.g., 1. Derived from GITHUB_RUN_ATTEMPT.
18: optional string github_run_attempt;
# A unique number for each workflow run within a repository, e.g., 19471190684. Derived from GITHUB_RUN_ID.
17: optional string github_run_id;
# Indicates if branch protections or rulesets are configured for the ref that triggered the workflow run. Derived from GITHUB_REF_PROTECTED.
20: optional bool github_ref_protected;
# A unique number for each attempt of a particular workflow run in a repository, e.g., 1. Derived from GITHUB_RUN_ATTEMPT.
18: optional string github_run_attempt;
# The fully-formed ref of the branch or tag that triggered the workflow run, e.g., refs/pull/133891/merge or refs/heads/main. Derived from GITHUB_REF.
21: optional string github_ref;
# Indicates if branch protections or rulesets are configured for the ref that triggered the workflow run. Derived from GITHUB_REF_PROTECTED.
20: optional bool github_ref_protected;
# The weight of the record according to current sampling rate
25: optional i64 weight;
# The fully-formed ref of the branch or tag that triggered the workflow run, e.g., refs/pull/133891/merge or refs/heads/main. Derived from GITHUB_REF.
21: optional string github_ref;
# The name of the current job. Derived from JOB_NAME, e.g., linux-jammy-py3.8-gcc11 / test (default, 3, 4, linux.2xlarge).
26: optional string github_job;
# The weight of the record according to current sampling rate
25: optional i64 weight;
# The GitHub user who triggered the job. Derived from GITHUB_TRIGGERING_ACTOR.
27: optional string github_triggering_actor;
# The name of the current job. Derived from JOB_NAME, e.g., linux-jammy-py3.8-gcc11 / test (default, 3, 4, linux.2xlarge).
26: optional string github_job;
# A unique number for each run of a particular workflow in a repository, e.g., 238742. Derived from GITHUB_RUN_NUMBER.
28: optional string github_run_number_str;
}
""", # noqa: B950
)
# The GitHub user who triggered the job. Derived from GITHUB_TRIGGERING_ACTOR.
27: optional string github_triggering_actor;
# A unique number for each run of a particular workflow in a repository, e.g., 238742. Derived from GITHUB_RUN_NUMBER.
28: optional string github_run_number_str;
}
""", # noqa: B950
)
class BenchmarkBase(ABC):
@ -237,10 +239,11 @@ class BenchmarkBase(ABC):
if self._enable_instruction_count:
r = self._count_instructions()
self.results.append((self.name(), "instruction_count", r))
scribe_log_torch_benchmark_compile_time(
name=self.name(),
instruction_count=r,
)
if log_to_scuba:
scribe_log_torch_benchmark_compile_time(
name=self.name(),
instruction_count=r,
)
if self._enable_compile_time_instruction_count:
r = self._count_compile_time_instructions()
@ -251,9 +254,10 @@ class BenchmarkBase(ABC):
r,
)
)
# TODO add a new field compile_time_instruction_count to the logger.
scribe_log_torch_benchmark_compile_time(
name=self.name(),
instruction_count=r,
)
if log_to_scuba:
# TODO add a new field compile_time_instruction_count to the logger.
scribe_log_torch_benchmark_compile_time(
name=self.name(),
instruction_count=r,
)
return self

7
benchmarks/dynamo/pr_time_benchmarks/check_results.py
View file

@ -219,6 +219,13 @@ def main():
f"There was some failures you can use the new reference expected result stored at path:"
f"{reference_expected_results_path} and printed above\n"
)
print(
"To reproduce locally follow the following instructions, note that absolute instructions count are going "
"to be different than on the CI, hence you might want to run locally with and without your change:\n"
"cd benchmarks/dynamo/pr_time_benchmarks/ \n"
"python benchmarks/BENCHMARK.py result.csv \n"
"note that BENCHMARK.py is the name of the file containing the failing benchmark."
)
sys.exit(1)
else:
print("All benchmarks passed")