saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add cpu/mem usage for perf metrics (#3947)

Browse source 
* add cpu/mem usage

* on comments

* on comments

* renaming

Co-authored-by: Ethan Tao <ettao@microsoft.com>
This commit is contained in:
ytaous 2020-05-15 12:29:40 -07:00 committed by GitHub
parent be003dbab7
commit bc441b7e5c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 36 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
cmake/onnxruntime_training.cmake
View file

@ -41,7 +41,21 @@ file(GLOB_RECURSE onnxruntime_training_runner_srcs
"${ORTTRAINING_SOURCE_DIR}/models/runner/*.h"
"${ORTTRAINING_SOURCE_DIR}/models/runner/*.cc"
)
add_library(onnxruntime_training_runner ${onnxruntime_training_runner_srcs})
# perf test utils
set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest)
set(onnxruntime_perf_test_src
"${onnxruntime_perf_test_src_dir}/utils.h")
if(WIN32)
list(APPEND onnxruntime_perf_test_src
"${onnxruntime_perf_test_src_dir}/windows/utils.cc")
else ()
list(APPEND onnxruntime_perf_test_src
"${onnxruntime_perf_test_src_dir}/posix/utils.cc")
endif()
add_library(onnxruntime_training_runner ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src})
add_dependencies(onnxruntime_training_runner ${onnxruntime_EXTERNAL_DEPENDENCIES} onnx onnxruntime_providers)
onnxruntime_add_include_to_target(onnxruntime_training_runner onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training)
@ -54,7 +68,7 @@ if(UNIX AND NOT APPLE)
target_compile_options(onnxruntime_training_runner PUBLIC "-Wno-maybe-uninitialized")
endif()
set_target_properties(onnxruntime_training_runner PROPERTIES FOLDER "ONNXRuntimeTest")
source_group(TREE ${ORTTRAINING_ROOT} FILES ${onnxruntime_training_runner_srcs})
source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src})
# MNIST

20
orttraining/orttraining/models/runner/training_runner.cc
View file

@ -17,6 +17,7 @@
#include "orttraining/core/graph/optimizer_graph_builder.h"
#include "orttraining/models/runner/training_util.h"
#include "single_include/nlohmann/json.hpp"
#include "test/perftest/utils.h"
using json = nlohmann::json;
@ -581,6 +582,11 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad
params_.mpi_context.world_rank == 0 &&
checkpoint_registry_ && params_.checkpoint_period > 0;
std::unique_ptr<perftest::utils::ICPUUsage> cpu_usage_calculator;
if (!params_.perf_output_dir.empty()) {
cpu_usage_calculator = perftest::utils::CreateICPUUsage();
}
if (test_data_loader) {
ORT_RETURN_IF_ERROR(test_data_loader->InitializeDataSetIndex(0));
}
@ -726,10 +732,13 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad
if (params_.perf_output_dir.empty()) {
printf("No perf output directory specified, skipping save of trained perf metrics.\n");
} else {
const short average_cpu_usage = cpu_usage_calculator->GetUsage();
const size_t peak_workingset_size = perftest::utils::GetPeakWorkingSetSize();
ORT_RETURN_IF_ERROR(Env::Default().CreateFolder(params_.perf_output_dir));
// saving json file
ORT_RETURN_IF_ERROR(SavePerfMetrics(number_of_batches, gradient_accumulation_step_count, weight_update_steps,
total_time, avg_time_per_batch, throughput, stabilized_throughput, mapped_dimensions));
total_time, avg_time_per_batch, throughput, stabilized_throughput, mapped_dimensions,
average_cpu_usage, peak_workingset_size));
}
std::cout << "Round: " << round_ << "\n"
@ -748,7 +757,8 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad
Status TrainingRunner::SavePerfMetrics(const size_t number_of_batches, const size_t gradient_accumulation_steps,
const size_t weight_update_steps, const double total_time,
const double avg_time_per_batch, const double throughput, const double stabilized_throughput,
const MapStringToString& mapped_dimensions) {
const MapStringToString& mapped_dimensions,
const short average_cpu_usage, const size_t peak_workingset_size) {
// populate metrics for reporting
json perf_metrics;
perf_metrics["Model"] = params_.model_type;
@ -789,10 +799,8 @@ Status TrainingRunner::SavePerfMetrics(const size_t number_of_batches, const siz
(seq_len.empty() ? "" : "_" + seq_len) + "_" + optimizer;
perf_metrics["DisplayName"] = display_name;
// TODO - add memory/cpu
//j["Memory"] = ;
//j["AvgCPU"] = ;
perf_metrics["Memory"] = peak_workingset_size;
perf_metrics["AvgCPU"] = average_cpu_usage;
//
// we will get date/time and commitId in post-run pipeline

3
orttraining/orttraining/models/runner/training_runner.h
View file

@ -213,7 +213,8 @@ class TrainingRunner {
Status SavePerfMetrics(const size_t number_of_batches, const size_t gradient_accumulation_steps,
const size_t weight_update_steps, const double total_time,
const double avg_time_per_batch, const double throughput, const double stabilized_throughput,
const MapStringToString& mapped_dimensions);
const MapStringToString& mapped_dimensions,
const short average_cpu_usage, const size_t peak_workingset_size);
size_t step_;
size_t round_;

24
tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java
View file

@ -140,17 +140,8 @@ public class App {
st.setFloat(++i, (float)(double) json_object.get("Throughput")); // examples/sec
st.setFloat(++i, (float)(double) json_object.get("StabilizedThroughput")); // examples/sec
st.setFloat(++i, (float)(double) json_object.get("TotalTime")); // secs
// TODO - remove "if" check later
if (json_object.get("AvgCPU") == null)
st.setNull(++i, Types.FLOAT);
else
st.setFloat(++i, (float)(double) json_object.get("AvgCPU"));
if (json_object.get("Memory") == null)
st.setNull(++i, Types.INTEGER);
else
st.setInt(++i, (int)(long) json_object.get("Memory")); // mb
st.setInt(++i, (int)(long) json_object.get("AvgCPU"));
st.setInt(++i, (int)((long) json_object.get("Memory") >> 20)); // mb
st.setString(++i, (String) json_object.get("RunConfig"));
// update section
@ -158,15 +149,8 @@ public class App {
st.setFloat(++i, (float)(double) json_object.get("Throughput")); // examples/sec
st.setFloat(++i, (float)(double) json_object.get("StabilizedThroughput")); // examples/sec
st.setFloat(++i, (float)(double) json_object.get("TotalTime")); // secs
if (json_object.get("AvgCPU") == null)
st.setNull(++i, Types.FLOAT);
else
st.setFloat(++i, (float)(double) json_object.get("AvgCPU"));
if (json_object.get("Memory") == null)
st.setNull(++i, Types.INTEGER);
else
st.setInt(++i, (int)(long) json_object.get("Memory")); // mb
st.setInt(++i, (int)((long) json_object.get("Memory") >> 20)); // mb
st.setString(++i, (String) json_object.get("RunConfig"));
st.executeUpdate();
} catch (Exception e) {