From 4bd7c2adf1d43960ed90f7b7ca0c04371be0c9b7 Mon Sep 17 00:00:00 2001 From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Date: Wed, 24 Nov 2021 13:10:20 -0800 Subject: [PATCH] Document a few features for rel 1.10 (#9836) * document threading hooks * move doc to a separate section * document cuda profiler numbers * move doc * specify cupti dependency * add example inplace * format code * format code * link to cxx api * minor fix on grammar * add new line * minor fix on grammar * minor fix on grammar * minor fix on grammar Co-authored-by: Randy Shuai --- .../CUDA-ExecutionProvider.md | 1 + docs/performance/tune-performance.md | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/docs/execution-providers/CUDA-ExecutionProvider.md b/docs/execution-providers/CUDA-ExecutionProvider.md index 359d3b5045..17e3c278fd 100644 --- a/docs/execution-providers/CUDA-ExecutionProvider.md +++ b/docs/execution-providers/CUDA-ExecutionProvider.md @@ -27,6 +27,7 @@ Please reference table below for official GPU packages dependencies for the ONNX |ONNX Runtime|CUDA|cuDNN|Notes| |---|---|---|---| +|1.10|11.4|8.2.4 (Linux)
8.2.2.26 (Windows)|libcudart 11.4.43
libcufft 10.5.2.100
libcurand 10.2.5.120
libcublasLt 11.6.1.51
libcublas 11.6.1.51
libcudnn 8.2.4
libcupti.so 2021.2.2| |1.9|11.4|8.2.4 (Linux)
8.2.2.26 (Windows)|libcudart 11.4.43
libcufft 10.5.2.100
libcurand 10.2.5.120
libcublasLt 11.6.1.51
libcublas 11.6.1.51
libcudnn 8.2.4
libcupti.so 2021.2.2| |1.8|11.0.3|8.0.4 (Linux)
8.0.2.39 (Windows)|libcudart 11.0.221
libcufft 10.2.1.245
libcurand 10.2.1.245
libcublasLt 11.2.0.252
libcublas 11.2.0.252
libcudnn 8.0.4
libcupti.so 2020.1.1| |1.7|11.0.3|8.0.4 (Linux)
8.0.2.39 (Windows)|libcudart 11.0.221
libcufft 10.2.1.245
libcurand 10.2.1.245
libcublasLt 11.2.0.252
libcublas 11.2.0.252
libcudnn 8.0.4| diff --git a/docs/performance/tune-performance.md b/docs/performance/tune-performance.md index 58c1b90651..4fbbbe4875 100644 --- a/docs/performance/tune-performance.md +++ b/docs/performance/tune-performance.md @@ -51,6 +51,19 @@ In both cases, you will get a JSON file which contains the detailed performance * Type chrome://tracing in the address bar * Load the generated JSON file +For CUDA EP, performance numbers from device will be attached to those from host. For example: +``` +{"cat":"Node", "name":"Add_1234", "dur":17, ...} +{"cat":"Kernel", "name":"ort_add_cuda_kernel", dur:33, ...} +``` +Here, "Add" operator from host initiated a CUDA kernel on device named "ort_add_cuda_kernel" which lasted for 33 microseconds. +If an operator called multiple kernels during execution, the performance numbers of those kernels will all be listed following the calling sequence: +``` +{"cat":"Node", "name":, ...} +{"cat":"Kernel", "name":, ...} +{"cat":"Kernel", "name":, ...} +``` + ## Using different Execution Providers To learn more about different Execution Providers, see [Reference: Execution Providers](../execution-providers). @@ -148,6 +161,61 @@ Memory consumption can be reduced between multiple sessions by configuring the s * Inter op num threads (used only when parallel execution is enabled) is not affected by OpenMP settings and should always be set using the ORT APIs. +### Custom threading callbacks +Occasionally, customers might prefer to use their own fine-tuned threads for multithreading, +hence ORT offers thread creation and joining callbacks by [C++ API](https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_cxx_api.h): + +``` + std::vector threads; + void* custom_thread_creation_options = nullptr; + // initialize custom_thread_creation_options + + // On thread pool creation, ORT calls CreateThreadCustomized to create a thread + OrtCustomThreadHandle CreateThreadCustomized(void* custom_thread_creation_options, OrtThreadWorkerFn work_loop, void* param) { + threads.push_back(std::thread(work_loop, param)); + // configure the thread by custom_thread_creation_options + return reinterpret_cast(threads.back().native_handle()); + } + + // On thread pool destruction, ORT calls JoinThreadCustomized for each created thread + void JoinThreadCustomized(OrtCustomThreadHandle handle) { + for (auto& t : threads) { + if (reinterpret_cast(t.native_handle()) == handle) { + // recycling resources ... + t.join(); + } + } + } + + int main(...) { + ... + Ort::Env ort_env; + Ort::SessionOptions session_options; + session_options.SetCustomCreateThreadFn(CreateThreadCustomized); + session_options.SetCustomThreadCreationOptions(&custom_thread_creation_options); + session_options.SetCustomJoinThreadFn(JoinThreadCustomized); + Ort::Session session(*ort_env, MODEL_URI, session_options); + ... + } +``` + +For global thread pool: + +``` + int main() { + const OrtApi* g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION); + OrtThreadingOptions* tp_options = nullptr; + g_ort->CreateThreadingOptions(&tp_options); + g_ort->SetGlobalCustomCreateThreadFn(tp_options, CreateThreadCustomized); + g_ort->SetGlobalCustomThreadCreationOptions(tp_options, &custom_thread_creation_options); + g_ort->SetGlobalCustomJoinThreadFn(tp_options, JoinThreadCustomized); + // disable per-session thread pool, create a session for inferencing + g_ort->ReleaseThreadingOptions(tp_options); + } +``` + +Note that CreateThreadCustomized and JoinThreadCustomized, once being set, will be applied to both ORT intra op and inter op thread pools uniformly. + ### Default CPU Execution Provider (MLAS) The default execution provider uses different knobs to control the thread number.