saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Execution Provider Profiler (#8406)

Browse source 
* implement cuda provider

* define profiler common

* call start after register

* add memcpy event

* add cuda correlation

* format code

* add cupti to test path

* switch to CUpti_ActivityKernel3

* reset cupti path

* fix test case

* fix trt pipeline

* add namespace

* format code

* exclude training from testing

* remove mutex
This commit is contained in:
RandySheriffH 2021-09-28 13:59:52 -07:00 committed by GitHub
parent 6f580f07de
commit 058108bef9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 434 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmake/onnxruntime_providers.cmake
View file

@ -362,8 +362,9 @@ if (onnxruntime_USE_CUDA)
endif()
add_dependencies(onnxruntime_providers_cuda onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies})
target_link_libraries(onnxruntime_providers_cuda PRIVATE cublas cudnn curand cufft ${ONNXRUNTIME_PROVIDERS_SHARED})
target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
target_link_directories(onnxruntime_providers_cuda PRIVATE ${onnxruntime_CUDA_HOME}/extras/CUPTI/lib64)
target_link_libraries(onnxruntime_providers_cuda PRIVATE cublas cudnn curand cufft cupti ${ONNXRUNTIME_PROVIDERS_SHARED})
target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${onnxruntime_CUDA_HOME}/extras/CUPTI/include)
# ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found
set_target_properties(onnxruntime_providers_cuda PROPERTIES LINKER_LANGUAGE CUDA)
set_target_properties(onnxruntime_providers_cuda PROPERTIES FOLDER "ONNXRuntime")

42
include/onnxruntime/core/common/logging/logging.h
View file

@ -12,6 +12,7 @@
#include <string>
#include "core/common/common.h"
#include "core/common/profiler_common.h"
#include "core/common/logging/capture.h"
#include "core/common/logging/severity.h"
@ -50,47 +51,6 @@
*/
namespace onnxruntime {
namespace profiling {
enum EventCategory {
SESSION_EVENT = 0,
NODE_EVENT,
EVENT_CATEGORY_MAX
};
/*
Event descriptions for the above session events.
*/
static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = {
"Session",
"Node"};
/*
Timing record for all events.
*/
struct EventRecord {
EventRecord(EventCategory category,
int process_id,
int thread_id,
std::string event_name,
long long time_stamp,
long long duration,
std::unordered_map<std::string, std::string>&& event_args) : cat(category),
pid(process_id),
tid(thread_id),
name(std::move(event_name)),
ts(time_stamp),
dur(duration),
args(event_args) {}
EventCategory cat;
int pid;
int tid;
std::string name;
long long ts;
long long dur;
std::unordered_map<std::string, std::string> args;
};
} // namespace profiling
namespace logging {

62
include/onnxruntime/core/common/profiler_common.h Normal file
View file

@ -0,0 +1,62 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
#include <unordered_map>
namespace onnxruntime {
namespace profiling {
enum EventCategory {
SESSION_EVENT = 0,
NODE_EVENT,
KERNEL_EVENT,
EVENT_CATEGORY_MAX
};
// Event descriptions for the above session events.
static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = {
"Session",
"Node",
"Kernel"};
// Timing record for all events.
struct EventRecord {
EventRecord(EventCategory category,
int process_id,
int thread_id,
std::string event_name,
long long time_stamp,
long long duration,
std::unordered_map<std::string, std::string>&& event_args) : cat(category),
pid(process_id),
tid(thread_id),
name(std::move(event_name)),
ts(time_stamp),
dur(duration),
args(event_args) {}
EventCategory cat;
int pid;
int tid;
std::string name;
long long ts;
long long dur;
std::unordered_map<std::string, std::string> args;
};
using Events = std::vector<EventRecord>;
//Execution Provider Profiler
class EpProfiler {
public:
virtual ~EpProfiler() = default;
virtual bool StartProfiling() = 0; // called when profiling starts
virtual void EndProfiling(TimePoint start_time, Events& events) = 0; // called when profiling ends, save all captures numbers to "events"
virtual void Start(uint64_t){}; // called before op start, accept an id as argument to identify the op
virtual void Stop(uint64_t){}; // called after op stop, accept an id as argument to identify the op
};
} //namespace profiling
} //namespace onnxruntime

6
include/onnxruntime/core/framework/execution_provider.h
View file

@ -19,6 +19,7 @@ class Node;
struct ComputeCapability;
class KernelRegistry;
class KernelRegistryManager;
} // namespace onnxruntime
#else
#include <memory>
@ -27,6 +28,7 @@ class KernelRegistryManager;
#include "core/framework/provider_options.h"
#include "core/framework/func_api.h"
#include "core/framework/allocatormgr.h"
#include "core/common/profiler_common.h"
namespace onnxruntime {
@ -266,6 +268,10 @@ class IExecutionProvider {
*/
virtual void RegisterAllocator(std::shared_ptr<AllocatorManager> allocator_manager);
virtual std::unique_ptr<profiling::EpProfiler> GetProfiler() {
return {};
}
private:
const std::string type_;
AllocatorMap allocators_;

27
onnxruntime/core/common/profiler.cc
View file

@ -19,9 +19,14 @@ profiling::Profiler::~Profiler() {
profiling::Profiler::~Profiler() {}
#endif
::onnxruntime::TimePoint profiling::Profiler::StartTime() const {
::onnxruntime::TimePoint profiling::Profiler::Start() {
ORT_ENFORCE(enabled_);
return std::chrono::high_resolution_clock::now();
auto start_time = std::chrono::high_resolution_clock::now();
auto ts = TimeDiffMicroSeconds(profiling_start_time_, start_time);
for (const auto& ep_profiler : ep_profilers_) {
ep_profiler->Start(ts);
}
return start_time;
}
void Profiler::Initialize(const logging::Logger* session_logger) {
@ -42,7 +47,10 @@ void Profiler::StartProfiling(const logging::Logger* custom_logger) {
enabled_ = true;
profile_with_logger_ = true;
custom_logger_ = custom_logger;
profiling_start_time_ = StartTime();
profiling_start_time_ = std::chrono::high_resolution_clock::now();
for (const auto& ep_profiler : ep_profilers_) {
ep_profiler->StartProfiling();
}
}
template <typename T>
@ -52,7 +60,10 @@ void Profiler::StartProfiling(const std::basic_string<T>& file_name) {
profile_stream_.open(file_name, std::ios::out | std::ios::trunc);
#endif
profile_stream_file_ = ToMBString(file_name);
profiling_start_time_ = StartTime();
profiling_start_time_ = std::chrono::high_resolution_clock::now();
for (const auto& ep_profiler : ep_profilers_) {
ep_profiler->StartProfiling();
}
}
template void Profiler::StartProfiling<char>(const std::basic_string<char>& file_name);
@ -85,6 +96,10 @@ void Profiler::EndTimeAndRecordEvent(EventCategory category,
}
}
}
for (const auto& ep_profiler : ep_profilers_) {
ep_profiler->Stop(ts);
}
}
std::string Profiler::EndProfiling() {
@ -103,6 +118,10 @@ std::string Profiler::EndProfiling() {
std::lock_guard<OrtMutex> lock(mutex_);
profile_stream_ << "[\n";
for (const auto& ep_profiler : ep_profilers_) {
ep_profiler->EndProfiling(profiling_start_time_, events_);
}
for (size_t i = 0; i < events_.size(); ++i) {
auto& rec = events_[i];
profile_stream_ << R"({"cat" : ")" << event_categor_names_[rec.cat] << "\",";

18
onnxruntime/core/common/profiler.h
View file

@ -9,6 +9,7 @@
#include <iostream>
#include <tuple>
#include "core/common/profiler_common.h"
#include "core/common/logging/logging.h"
#include "core/platform/ort_mutex.h"
@ -49,9 +50,9 @@ class Profiler {
void StartProfiling(const std::basic_string<T>& file_name);
/*
Produce current time point for any profiling action.
Start profiling and return current time point.
*/
TimePoint StartTime() const;
TimePoint Start();
/*
Whether data collection and output from this profiler is enabled.
@ -107,6 +108,15 @@ class Profiler {
static void SetGlobalMaxNumEvents(size_t new_max_num_events) {
global_max_num_events_.store(new_max_num_events);
}
void AddEpProfilers(std::unique_ptr<EpProfiler> ep_profiler) {
if (ep_profiler) {
ep_profilers_.push_back(std::move(ep_profiler));
if (enabled_) {
ep_profilers_.back()->StartProfiling();
}
}
}
private:
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Profiler);
@ -135,7 +145,7 @@ class Profiler {
const logging::Logger* session_logger_{nullptr};
const logging::Logger* custom_logger_{nullptr};
TimePoint profiling_start_time_;
std::vector<EventRecord> events_;
Events events_;
bool max_events_reached{false};
bool profile_with_logger_{false};
const size_t max_num_events_{global_max_num_events_.load()};
@ -143,6 +153,8 @@ class Profiler {
#ifdef ENABLE_STATIC_PROFILER_INSTANCE
static Profiler* instance_;
#endif
std::vector<std::unique_ptr<EpProfiler>> ep_profilers_;
};
} // namespace profiling

8
onnxruntime/core/framework/orttraining_partial_executor.cc
View file

@ -144,7 +144,7 @@ Status PartialExecutor::Execute(const SessionState& session_state, const std::ve
size_t total_output_sizes = 0;
if (is_profiler_enabled) {
tp = session_state.Profiler().StartTime();
tp = session_state.Profiler().Start();
}
ExecutionFrame& frame = state_.GetExecutionFrame(feed_mlvalue_idxs, feeds, fetch_mlvalue_idxs, fetches,
@ -256,7 +256,7 @@ Status PartialExecutor::Execute(const SessionState& session_state, const std::ve
// TODO: log kernel outputs?
if (is_profiler_enabled) {
sync_time_begin = session_state.Profiler().StartTime();
sync_time_begin = session_state.Profiler().Start();
}
// sync before compute
@ -311,7 +311,7 @@ Status PartialExecutor::Execute(const SessionState& session_state, const std::ve
// call compute on the kernel
VLOGS(logger, 1) << "Computing kernel: " << node_name_for_profiling;
kernel_begin_time = session_state.Profiler().StartTime();
kernel_begin_time = session_state.Profiler().Start();
// Calculate total input sizes for this operation.
CalculateTotalInputSizes(&op_kernel_context, p_op_kernel,
@ -399,7 +399,7 @@ Status PartialExecutor::Execute(const SessionState& session_state, const std::ve
concurrency::ThreadPool::StopProfiling(
session_state.GetThreadPool())},
});
sync_time_begin = session_state.Profiler().StartTime();
sync_time_begin = session_state.Profiler().Start();
}
// sync after compute for outputs

8
onnxruntime/core/framework/parallel_executor.cc
View file

@ -35,7 +35,7 @@ Status ParallelExecutor::Execute(const SessionState& session_state, const std::v
TimePoint tp;
const bool is_profiler_enabled = session_state.Profiler().IsEnabled();
if (is_profiler_enabled) {
tp = session_state.Profiler().StartTime();
tp = session_state.Profiler().Start();
}
root_frame_ = std::make_unique<ExecutionFrame>(feed_mlvalue_idxs, feeds, fetch_mlvalue_idxs, fetches,
@ -142,7 +142,7 @@ Status ParallelExecutor::RunNodeAsync(size_t p_node_index,
OpKernelContextInternal op_kernel_context(session_state, *root_frame_, *p_op_kernel, logger, terminate_flag_);
if (f_profiler_enabled) {
sync_time_begin = session_state.Profiler().StartTime();
sync_time_begin = session_state.Profiler().Start();
}
// sync before compute
int queue_id = p_op_kernel->KernelDef().ExecQueueId();
@ -183,7 +183,7 @@ Status ParallelExecutor::RunNodeAsync(size_t p_node_index,
sync_time_begin,
{{"op_name", p_op_kernel->KernelDef().OpName()}});
concurrency::ThreadPool::StartProfiling(session_state.GetThreadPool());
kernel_begin_time = session_state.Profiler().StartTime();
kernel_begin_time = session_state.Profiler().Start();
}
// call compute on the kernel
@ -223,7 +223,7 @@ Status ParallelExecutor::RunNodeAsync(size_t p_node_index,
{"provider", p_op_kernel->KernelDef().Provider()},
{"thread_scheduling_stats", concurrency::ThreadPool::StopProfiling(session_state.GetThreadPool())}});
sync_time_begin = session_state.Profiler().StartTime();
sync_time_begin = session_state.Profiler().Start();
}
// sync after compute for outputs
if (exec_plan.NodeHasFence(node_index)) {

8
onnxruntime/core/framework/sequential_executor.cc
View file

@ -136,7 +136,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
size_t total_output_sizes = 0;
if (is_profiler_enabled) {
tp = session_state.Profiler().StartTime();
tp = session_state.Profiler().Start();
}
ExecutionFrame frame{feed_mlvalue_idxs, feeds, fetch_mlvalue_idxs, fetches, fetch_allocators, session_state};
@ -241,7 +241,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
OpKernelContextInternal op_kernel_context(session_state, frame, *p_op_kernel, logger, terminate_flag_);
// TODO: log kernel outputs?
if (is_profiler_enabled) {
sync_time_begin = session_state.Profiler().StartTime();
sync_time_begin = session_state.Profiler().Start();
}
// sync before compute
@ -296,7 +296,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
// call compute on the kernel
VLOGS(logger, 1) << "Computing kernel: " << node_name_for_profiling;
kernel_begin_time = session_state.Profiler().StartTime();
kernel_begin_time = session_state.Profiler().Start();
// Calculate total input sizes for this operation.
CalculateTotalInputSizes(&op_kernel_context, p_op_kernel,
@ -378,7 +378,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
{"output_size", std::to_string(total_output_sizes)},
{"thread_scheduling_stats", concurrency::ThreadPool::StopProfiling(session_state.GetThreadPool())},
});
sync_time_begin = session_state.Profiler().StartTime();
sync_time_begin = session_state.Profiler().Start();
}
// sync after compute for outputs

13
onnxruntime/core/providers/cuda/cuda_execution_provider.cc
View file

@ -8,6 +8,7 @@
#include "core/providers/cuda/cuda_fence.h"
#include "core/providers/cuda/cuda_fwd.h"
#include "core/providers/cuda/gpu_data_transfer.h"
#include "core/providers/cuda/cuda_profiler.h"
#ifndef DISABLE_CONTRIB_OPS
#include "contrib_ops/cuda/cuda_contrib_kernels.h"
@ -214,6 +215,10 @@ CUDAExecutionProvider::~CUDAExecutionProvider() {
}
}
std::unique_ptr<profiling::EpProfiler> CUDAExecutionProvider::GetProfiler() {
return std::make_unique<profiling::CudaProfiler>();
}
CUDAExecutionProvider::PerThreadContext& CUDAExecutionProvider::GetPerThreadContext() const {
const auto& per_thread_context_cache = PerThreadContextCache();
@ -2268,7 +2273,6 @@ CUDAExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
// These are usually shape related computation subgraphs
// Following logic can be extended for other EPs
std::unordered_set<NodeIndex> cpu_nodes = GetCpuPreferredNodes(graph, Type(), kernel_registries, candidates);
std::vector<std::unique_ptr<ComputeCapability>> result;
for (auto& node_index : candidates) {
if (cpu_nodes.count(node_index) > 0)
@ -2278,6 +2282,13 @@ CUDAExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
sub_graph->Nodes().push_back(node_index);
result.push_back(ComputeCapability::Create(std::move(sub_graph)));
}
/*
std::vector<std::unique_ptr<ComputeCapability>> result;
for (auto& node_index : candidates) {
auto sub_graph = IndexedSubGraph::Create();
sub_graph->Nodes().push_back(node_index);
result.push_back(ComputeCapability::Create(std::move(sub_graph)));
}*/
return result;
}

2
onnxruntime/core/providers/cuda/cuda_execution_provider.h
View file

@ -92,6 +92,8 @@ class CUDAExecutionProvider : public IExecutionProvider {
static AllocatorPtr CreateCudaAllocator(OrtDevice::DeviceId device_id, size_t cuda_mem_limit, ArenaExtendStrategy arena_extend_strategy,
CUDAExecutionProviderExternalAllocatorInfo external_alloc_info, OrtArenaCfg* arena_cfg);
std::unique_ptr<profiling::EpProfiler> GetProfiler() override;
private:
CUDAExecutionProviderInfo info_;
cudaDeviceProp device_prop_;

199
onnxruntime/core/providers/cuda/cuda_profiler.cc Normal file
View file

@ -0,0 +1,199 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !(defined(USE_ROCM) || defined(ENABLE_TRAINING))
#include "cuda_profiler.h"
#include <map>
#include <string>
#include <iostream>
namespace onnxruntime {
namespace profiling {
auto KEVENT = onnxruntime::profiling::KERNEL_EVENT;
std::atomic_flag CudaProfiler::enabled{0};
std::vector<CudaProfiler::KernelStat> CudaProfiler::stats;
std::unordered_map<uint32_t, uint64_t> CudaProfiler::id_map;
#define BUF_SIZE (32 * 1024)
#define ALIGN_SIZE (8)
#define ALIGN_BUFFER(buffer, align) \
(((uintptr_t)(buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) : (buffer))
#define DUR(s, e) ((e - s) / 1000)
static const char* GetMemcpyKindString(CUpti_ActivityMemcpyKind kind) {
switch (kind) {
case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD:
return "MemcpyHostToDevice";
case CUPTI_ACTIVITY_MEMCPY_KIND_DTOH:
return "MemcpyDeviceToHost";
case CUPTI_ACTIVITY_MEMCPY_KIND_HTOA:
return "MemcpyHostToDeviceArray";
case CUPTI_ACTIVITY_MEMCPY_KIND_ATOH:
return "MemcpyDeviceArrayToHost";
case CUPTI_ACTIVITY_MEMCPY_KIND_ATOA:
return "MemcpyDeviceArrayToDeviceArray";
case CUPTI_ACTIVITY_MEMCPY_KIND_ATOD:
return "MemcpyDeviceArrayToDevice";
case CUPTI_ACTIVITY_MEMCPY_KIND_DTOA:
return "MemcpyDeviceToDeviceArray";
case CUPTI_ACTIVITY_MEMCPY_KIND_DTOD:
return "MemcpyDeviceToDevice";
case CUPTI_ACTIVITY_MEMCPY_KIND_HTOH:
return "MemcpyHostToHost";
default:
break;
}
return "<unknown>";
}
void CUPTIAPI CudaProfiler::BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) {
uint8_t* bfr = (uint8_t*)malloc(BUF_SIZE + ALIGN_SIZE);
*size = BUF_SIZE;
*buffer = ALIGN_BUFFER(bfr, ALIGN_SIZE);
*maxNumRecords = 0;
}
void CUPTIAPI CudaProfiler::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t validSize) {
CUptiResult status;
CUpti_Activity* record = NULL;
if (validSize > 0) {
do {
status = cuptiActivityGetNextRecord(buffer, validSize, &record);
if (status == CUPTI_SUCCESS) {
if (CUPTI_ACTIVITY_KIND_KERNEL == record->kind) {
CUpti_ActivityKernel3* kernel = (CUpti_ActivityKernel3*)record;
stats.push_back({kernel->name, kernel->streamId,
kernel->gridX, kernel->gridY, kernel->gridZ,
kernel->blockX, kernel->blockY, kernel->blockZ,
static_cast<int64_t>(kernel->start),
static_cast<int64_t>(kernel->end),
kernel->correlationId});
} else if (CUPTI_ACTIVITY_KIND_MEMCPY == record->kind) {
CUpti_ActivityMemcpy3* mmcpy = (CUpti_ActivityMemcpy3*)record;
stats.push_back({GetMemcpyKindString((CUpti_ActivityMemcpyKind)mmcpy->copyKind),
mmcpy->streamId, -1, -1, -1, -1, -1, -1,
static_cast<int64_t>(mmcpy->start),
static_cast<int64_t>(mmcpy->end),
mmcpy->correlationId});
} else if (CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION == record->kind) {
auto correlation = reinterpret_cast<const CUpti_ActivityExternalCorrelation*>(record);
id_map.insert({correlation->correlationId, correlation->externalId});
}
} else if (status == CUPTI_ERROR_MAX_LIMIT_REACHED) {
break;
}
} while (1);
}
free(buffer);
}
bool CudaProfiler::StartProfiling() {
if (!enabled.test_and_set()) {
if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME) == CUPTI_SUCCESS &&
cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER) == CUPTI_SUCCESS &&
cuptiActivityEnable(CUPTI_ACTIVITY_KIND_KERNEL) == CUPTI_SUCCESS &&
cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY) == CUPTI_SUCCESS &&
cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION) == CUPTI_SUCCESS &&
cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) {
initialized_ = true;
return true;
} else {
DisableEvents();
enabled.clear();
return false;
}
}
return false;
}
void CudaProfiler::EndProfiling(TimePoint start_time, Events& events) {
std::map<uint64_t, std::vector<EventRecord>> event_map;
if (initialized_) {
DisableEvents();
cuptiActivityFlushAll(1);
int64_t profiling_start = std::chrono::duration_cast<std::chrono::nanoseconds>(start_time.time_since_epoch()).count();
for (const auto& stat : stats) {
std::initializer_list<std::pair<std::string, std::string>> args = {{"op_name", ""},
{"stream", std::to_string(stat.stream_)},
{"grid_x", std::to_string(stat.grid_x_)},
{"grid_y", std::to_string(stat.grid_y_)},
{"grid_z", std::to_string(stat.grid_z_)},
{"block_x", std::to_string(stat.block_x_)},
{"block_y", std::to_string(stat.block_y_)},
{"block_z", std::to_string(stat.block_z_)}};
EventRecord event{
KEVENT, -1, -1, stat.name_, DUR(profiling_start, stat.stop_), DUR(stat.start_, stat.stop_), {args.begin(), args.end()}};
auto ts = id_map[stat.correlation_id];
if (event_map.find(ts) == event_map.end()) {
event_map.insert({ts, {event}});
} else {
event_map[ts].push_back(std::move(event));
}
}
auto insert_iter = events.begin();
for (auto& map_iter : event_map) {
auto ts = static_cast<long long>(map_iter.first);
while (insert_iter != events.end() && insert_iter->ts < ts) {
insert_iter++;
}
if (insert_iter != events.end() && insert_iter != events.begin() && insert_iter->ts > ts) {
insert_iter--;
}
if (insert_iter != events.end() && insert_iter->ts == ts) {
for (auto& evt_iter : map_iter.second) {
evt_iter.args["op_name"] = insert_iter->args["op_name"];
}
}
insert_iter = events.insert(insert_iter, map_iter.second.begin(), map_iter.second.end());
while (insert_iter != events.end() && insert_iter->cat == EventCategory::KERNEL_EVENT) {
insert_iter++;
}
}
cuptiFinalize();
Clear();
} //if initialized
}
CudaProfiler::~CudaProfiler() {
if (initialized_) {
DisableEvents();
cuptiFinalize();
Clear();
}
}
void CudaProfiler::Start(uint64_t id) {
if (initialized_) {
cuptiActivityPushExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, id);
}
}
void CudaProfiler::Stop(uint64_t) {
if (initialized_) {
uint64_t last_id{0};
cuptiActivityPopExternalCorrelationId(CUPTI_EXTERNAL_CORRELATION_KIND_UNKNOWN, &last_id);
}
}
void CudaProfiler::DisableEvents() {
cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION);
cuptiActivityDisable(CUPTI_ACTIVITY_KIND_KERNEL);
cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY);
cuptiActivityDisable(CUPTI_ACTIVITY_KIND_DRIVER);
cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME);
}
void CudaProfiler::Clear() {
if (initialized_) {
id_map.clear();
stats.clear();
initialized_ = false;
enabled.clear();
}
}
} // namespace profiling
} // namespace onnxruntime
#endif

81
onnxruntime/core/providers/cuda/cuda_profiler.h Normal file
View file

@ -0,0 +1,81 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/profiler_common.h"
#if defined(USE_ROCM) || defined(ENABLE_TRAINING)
namespace onnxruntime {
namespace profiling {
class CudaProfiler final : public EpProfiler {
public:
bool StartProfiling() override { return true; }
void EndProfiling(TimePoint, Events&) override{};
void Start(uint64_t) override{};
void Stop(uint64_t) override{};
};
}
}
#else
#include "core/platform/ort_mutex.h"
#include <cupti.h>
#include <mutex>
#include <vector>
namespace onnxruntime {
namespace profiling {
using Events = std::vector<onnxruntime::profiling::EventRecord>;
class CudaProfiler final : public EpProfiler {
public:
CudaProfiler() = default;
CudaProfiler(const CudaProfiler&) = delete;
CudaProfiler& operator=(const CudaProfiler&) = delete;
CudaProfiler(CudaProfiler&& cuda_profiler) noexcept {
initialized_ = cuda_profiler.initialized_;
cuda_profiler.initialized_ = false;
}
CudaProfiler& operator=(CudaProfiler&& cuda_profiler) noexcept {
initialized_ = cuda_profiler.initialized_;
cuda_profiler.initialized_ = false;
return *this;
}
~CudaProfiler();
bool StartProfiling() override;
void EndProfiling(TimePoint start_time, Events& events) override;
void Start(uint64_t) override;
void Stop(uint64_t) override;
private:
static void CUPTIAPI BufferRequested(uint8_t**, size_t*, size_t*);
static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t*, size_t, size_t);
struct KernelStat {
std::string name_ = {};
uint32_t stream_ = 0;
int32_t grid_x_ = 0;
int32_t grid_y_ = 0;
int32_t grid_z_ = 0;
int32_t block_x_ = 0;
int32_t block_y_ = 0;
int32_t block_z_ = 0;
int64_t start_ = 0;
int64_t stop_ = 0;
uint32_t correlation_id = 0;
};
static std::atomic_flag enabled;
static std::vector<KernelStat> stats;
static std::unordered_map<uint32_t, uint64_t> id_map;
void DisableEvents();
void Clear();
bool initialized_ = false;
};
} // namespace profiling
} // namespace onnxruntime
#endif

7
onnxruntime/core/session/inference_session.cc
View file

@ -533,6 +533,7 @@ common::Status InferenceSession::RegisterExecutionProvider(const std::shared_ptr
}
p_exec_provider->SetLogger(session_logger_);
session_profiler_.AddEpProfilers(p_exec_provider->GetProfiler());
return execution_providers_.Add(provider_type, std::move(p_exec_provider));
}
@ -631,7 +632,7 @@ common::Status InferenceSession::Load(std::function<common::Status(std::shared_p
Status status = Status::OK();
TimePoint tp;
if (session_profiler_.IsEnabled()) {
tp = session_profiler_.StartTime();
tp = session_profiler_.Start();
}
ORT_TRY {
std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
@ -1235,7 +1236,7 @@ common::Status InferenceSession::Initialize() {
Status status = Status::OK();
TimePoint tp;
if (session_profiler_.IsEnabled()) {
tp = session_profiler_.StartTime();
tp = session_profiler_.Start();
}
ORT_TRY {
@ -1742,7 +1743,7 @@ Status InferenceSession::Run(const RunOptions& run_options,
const std::vector<OrtDevice>* p_fetches_device_info) {
TimePoint tp;
if (session_profiler_.IsEnabled()) {
tp = session_profiler_.StartTime();
tp = session_profiler_.Start();
}
#ifdef ONNXRUNTIME_ENABLE_INSTRUMENT

8
onnxruntime/test/framework/inference_session_test.cc
View file

@ -643,11 +643,19 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) {
ASSERT_TRUE(lines[size - 1].find("]") != string::npos);
std::vector<std::string> tags = {"pid", "dur", "ts", "ph", "X", "name", "args"};
bool has_kernel_info = false;
for (size_t i = 1; i < size - 1; ++i) {
for (auto& s : tags) {
ASSERT_TRUE(lines[i].find(s) != string::npos);
has_kernel_info = has_kernel_info || lines[i].find("Kernel") != string::npos &&
lines[i].find("stream") != string::npos &&
lines[i].find("block_x") != string::npos;
}
}
#if defined(USE_CUDA) && !defined(ENABLE_TRAINING)
ASSERT_TRUE(has_kernel_info);
#endif
}
TEST(InferenceSessionTests, CheckRunProfilerWithStartProfile) {

4
onnxruntime/test/python/onnxruntime_test_python.py
View file

@ -599,10 +599,10 @@ class TestInferenceSession(unittest.TestCase):
with open(profile_file) as f:
lines = f.readlines()
self.assertTrue('[' in lines[0])
for i in range(1, 8):
for i in range(1, len(lines)-1):
for tag in tags:
self.assertTrue(tag in lines[i])
self.assertTrue(']' in lines[8])
self.assertTrue(']' in lines[-1])
def testProfilerGetStartTimeNs(self):
def getSingleSessionProfilingStartTime():

2
tools/ci_build/github/windows/setup_env_cuda.bat
View file

@ -1,2 +1,2 @@
set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin;%PATH%
set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\extras\CUPTI\lib64;%PATH%
set GRADLE_OPTS=-Dorg.gradle.daemon=false

2
tools/ci_build/github/windows/setup_env_cuda_11.bat
View file

@ -1,2 +1,2 @@
set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;%PATH%
set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;%PATH%
set GRADLE_OPTS=-Dorg.gradle.daemon=false

2
tools/ci_build/github/windows/setup_env_gpu.bat
View file

@ -1,2 +1,2 @@
set PATH=C:\azcopy;C:\local\TensorRT-8.0.3.4.Windows10.x86_64.cuda-11.3.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin;%PATH%
set PATH=C:\azcopy;C:\local\TensorRT-8.0.3.4.Windows10.x86_64.cuda-11.3.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;%PATH%
set GRADLE_OPTS=-Dorg.gradle.daemon=false

2
tools/ci_build/github/windows/setup_env_trt.bat
View file

@ -1,2 +1,2 @@
set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda\bin;%PATH%
set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda\bin;%PATH%
set GRADLE_OPTS=-Dorg.gradle.daemon=false