diff --git a/cmake/onnxruntime_training.cmake b/cmake/onnxruntime_training.cmake index a2d81e0056..505ea8d4fd 100644 --- a/cmake/onnxruntime_training.cmake +++ b/cmake/onnxruntime_training.cmake @@ -41,7 +41,21 @@ file(GLOB_RECURSE onnxruntime_training_runner_srcs "${ORTTRAINING_SOURCE_DIR}/models/runner/*.h" "${ORTTRAINING_SOURCE_DIR}/models/runner/*.cc" ) -add_library(onnxruntime_training_runner ${onnxruntime_training_runner_srcs}) + +# perf test utils +set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) +set(onnxruntime_perf_test_src +"${onnxruntime_perf_test_src_dir}/utils.h") + +if(WIN32) + list(APPEND onnxruntime_perf_test_src + "${onnxruntime_perf_test_src_dir}/windows/utils.cc") +else () + list(APPEND onnxruntime_perf_test_src + "${onnxruntime_perf_test_src_dir}/posix/utils.cc") +endif() + +add_library(onnxruntime_training_runner ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src}) add_dependencies(onnxruntime_training_runner ${onnxruntime_EXTERNAL_DEPENDENCIES} onnx onnxruntime_providers) onnxruntime_add_include_to_target(onnxruntime_training_runner onnxruntime_common onnx onnx_proto protobuf::libprotobuf onnxruntime_training) @@ -54,7 +68,7 @@ if(UNIX AND NOT APPLE) target_compile_options(onnxruntime_training_runner PUBLIC "-Wno-maybe-uninitialized") endif() set_target_properties(onnxruntime_training_runner PROPERTIES FOLDER "ONNXRuntimeTest") -source_group(TREE ${ORTTRAINING_ROOT} FILES ${onnxruntime_training_runner_srcs}) +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_training_runner_srcs} ${onnxruntime_perf_test_src}) # MNIST diff --git a/orttraining/orttraining/models/runner/training_runner.cc b/orttraining/orttraining/models/runner/training_runner.cc index b638a22f3e..24ffc90bb0 100644 --- a/orttraining/orttraining/models/runner/training_runner.cc +++ b/orttraining/orttraining/models/runner/training_runner.cc @@ -17,6 +17,7 @@ #include "orttraining/core/graph/optimizer_graph_builder.h" #include "orttraining/models/runner/training_util.h" #include "single_include/nlohmann/json.hpp" +#include "test/perftest/utils.h" using json = nlohmann::json; @@ -581,6 +582,11 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad params_.mpi_context.world_rank == 0 && checkpoint_registry_ && params_.checkpoint_period > 0; + std::unique_ptr cpu_usage_calculator; + if (!params_.perf_output_dir.empty()) { + cpu_usage_calculator = perftest::utils::CreateICPUUsage(); + } + if (test_data_loader) { ORT_RETURN_IF_ERROR(test_data_loader->InitializeDataSetIndex(0)); } @@ -726,10 +732,13 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad if (params_.perf_output_dir.empty()) { printf("No perf output directory specified, skipping save of trained perf metrics.\n"); } else { + const short average_cpu_usage = cpu_usage_calculator->GetUsage(); + const size_t peak_workingset_size = perftest::utils::GetPeakWorkingSetSize(); ORT_RETURN_IF_ERROR(Env::Default().CreateFolder(params_.perf_output_dir)); // saving json file ORT_RETURN_IF_ERROR(SavePerfMetrics(number_of_batches, gradient_accumulation_step_count, weight_update_steps, - total_time, avg_time_per_batch, throughput, stabilized_throughput, mapped_dimensions)); + total_time, avg_time_per_batch, throughput, stabilized_throughput, mapped_dimensions, + average_cpu_usage, peak_workingset_size)); } std::cout << "Round: " << round_ << "\n" @@ -748,7 +757,8 @@ Status TrainingRunner::TrainingLoop(IDataLoader& training_data_loader, IDataLoad Status TrainingRunner::SavePerfMetrics(const size_t number_of_batches, const size_t gradient_accumulation_steps, const size_t weight_update_steps, const double total_time, const double avg_time_per_batch, const double throughput, const double stabilized_throughput, - const MapStringToString& mapped_dimensions) { + const MapStringToString& mapped_dimensions, + const short average_cpu_usage, const size_t peak_workingset_size) { // populate metrics for reporting json perf_metrics; perf_metrics["Model"] = params_.model_type; @@ -789,10 +799,8 @@ Status TrainingRunner::SavePerfMetrics(const size_t number_of_batches, const siz (seq_len.empty() ? "" : "_" + seq_len) + "_" + optimizer; perf_metrics["DisplayName"] = display_name; - - // TODO - add memory/cpu - //j["Memory"] = ; - //j["AvgCPU"] = ; + perf_metrics["Memory"] = peak_workingset_size; + perf_metrics["AvgCPU"] = average_cpu_usage; // // we will get date/time and commitId in post-run pipeline diff --git a/orttraining/orttraining/models/runner/training_runner.h b/orttraining/orttraining/models/runner/training_runner.h index c0be37949e..2ebad08a90 100644 --- a/orttraining/orttraining/models/runner/training_runner.h +++ b/orttraining/orttraining/models/runner/training_runner.h @@ -213,7 +213,8 @@ class TrainingRunner { Status SavePerfMetrics(const size_t number_of_batches, const size_t gradient_accumulation_steps, const size_t weight_update_steps, const double total_time, const double avg_time_per_batch, const double throughput, const double stabilized_throughput, - const MapStringToString& mapped_dimensions); + const MapStringToString& mapped_dimensions, + const short average_cpu_usage, const size_t peak_workingset_size); size_t step_; size_t round_; diff --git a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java b/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java index 80b20535d7..c71d772727 100644 --- a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java +++ b/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java @@ -140,17 +140,8 @@ public class App { st.setFloat(++i, (float)(double) json_object.get("Throughput")); // examples/sec st.setFloat(++i, (float)(double) json_object.get("StabilizedThroughput")); // examples/sec st.setFloat(++i, (float)(double) json_object.get("TotalTime")); // secs - // TODO - remove "if" check later - if (json_object.get("AvgCPU") == null) - st.setNull(++i, Types.FLOAT); - else - st.setFloat(++i, (float)(double) json_object.get("AvgCPU")); - - if (json_object.get("Memory") == null) - st.setNull(++i, Types.INTEGER); - else - st.setInt(++i, (int)(long) json_object.get("Memory")); // mb - + st.setInt(++i, (int)(long) json_object.get("AvgCPU")); + st.setInt(++i, (int)((long) json_object.get("Memory") >> 20)); // mb st.setString(++i, (String) json_object.get("RunConfig")); // update section @@ -158,15 +149,8 @@ public class App { st.setFloat(++i, (float)(double) json_object.get("Throughput")); // examples/sec st.setFloat(++i, (float)(double) json_object.get("StabilizedThroughput")); // examples/sec st.setFloat(++i, (float)(double) json_object.get("TotalTime")); // secs - if (json_object.get("AvgCPU") == null) - st.setNull(++i, Types.FLOAT); - else - st.setFloat(++i, (float)(double) json_object.get("AvgCPU")); - - if (json_object.get("Memory") == null) - st.setNull(++i, Types.INTEGER); - else - st.setInt(++i, (int)(long) json_object.get("Memory")); // mb + st.setInt(++i, (int)((long) json_object.get("Memory") >> 20)); // mb + st.setString(++i, (String) json_object.get("RunConfig")); st.executeUpdate(); } catch (Exception e) {