saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

collect the last few iteration latency for throuput calculation (#3766)

Browse source
This commit is contained in:
pengwa 2020-05-01 13:24:17 +08:00 committed by GitHub
parent ad63e2593d
commit 98b97be635
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
orttraining/orttraining/models/runner/training_runner.cc
View file

@ -253,6 +253,11 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad
const auto step_start = step_;
const auto weight_update_step_count_start = weight_update_step_count_;
// how many steps at last we used for stabilized perf benchmarking.
const size_t stabilized_perf_total_step_count = std::min(static_cast<size_t>(128), params_.num_train_steps);
const size_t stabilized_perf_start_step = params_.num_train_steps - stabilized_perf_total_step_count;
double stabilized_total_time{0};
while (step_ < params_.num_train_steps) {
for (size_t shard_it = 0; shard_it < num_shards_to_visit; ++shard_it) {
auto training_data = training_data_loader.CurrentDataSet();
@ -333,6 +338,9 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> duration_seconds = end - start;
total_time += duration_seconds.count();
if (step_ >= stabilized_perf_start_step) {
stabilized_total_time += duration_seconds.count();
}
// Print some info when reaching the end of the batch.
printf("Round %d, Step: %d, epoch: %d, batch: %d/%d, shard_iteration: %d/%d, time: %.2f ms, throughput: %.2f ex/sec \n",
@ -395,7 +403,9 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad
<< "Weight Update Steps: " << (weight_update_step_count_ - weight_update_step_count_start) << "\n"
<< "Total Running Time: " << total_time << " Seconds \n"
<< "Average Running Time Per Batch: " << total_time / (step_ - step_start) * 1000 << " ms\n"
<< "Throughput: " << params_.batch_size * (step_ - step_start) / total_time << " Examples / Second\n";
<< "Throughput: " << params_.batch_size * (step_ - step_start) / total_time << " Examples / Second\n"
<< "Stabilized Throughput: " << params_.batch_size / (stabilized_total_time / stabilized_perf_total_step_count)
<< " Examples / Second\n";
return Status::OK();
}