From 97d0a46afc240d9f6dc3ef3b42f685db6b0252d8 Mon Sep 17 00:00:00 2001 From: Tommy Trimeloni Date: Wed, 21 Aug 2019 14:04:53 -0700 Subject: [PATCH] nGraph EP Optimizations (#1630) * Added check for unnecessary function initializations, and removed lock from unneeded areas of code. * Added LRU cache to EP. * Bugfixes for nGraph EP Optimization PR * Changed default cache size to 500 and refactored mutex readability. * Fixed unsafe environmental variable fetch for Windows. * Cleaned up Windows environment functions and cleaned up mutexes. --- .../core/providers/ngraph/ngraph_custom_op.cc | 53 +++++++++++++++++-- .../core/providers/ngraph/ngraph_custom_op.h | 3 +- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc index 9b4d572624..326e878cbc 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc @@ -25,6 +25,8 @@ namespace onnxruntime { namespace ngraph_ep { +#define NGRAPH_EP_LRU_CACHE_DEFAULT_SIZE 500 + static bool check_ngraph_dump_ops() { #ifdef _WIN32 size_t env_name_len = 0; @@ -80,7 +82,45 @@ void NGRAPHCustomOp::Initialize(const OrtCustomOpApi* api, OrtKernelContext* con uniq_input_shape.append(reinterpret_cast(tensor_shape.data()), ndim * sizeof(int64_t)); } - auto it = ng_exe_map_.insert({uniq_input_shape, nullptr}); //TODO: Limit the size of map with configurable size. + // Get cache size from environment + std::string tempSize; + #ifdef _WIN32 + char *buf{nullptr}; + size_t bufSize = 0; + if (!_dupenv_s(&buf, &bufSize, "ONNXRUNTIME_NGRAPH_LRU_CACHE_SIZE") && buf) { + tempSize = buf; + free(buf); + } + #else + if (std::getenv("ONNXRUNTIME_NGRAPH_LRU_CACHE_SIZE")) { + tempSize = std::getenv("ONNXRUNTIME_NGRAPH_LRU_CACHE_SIZE"); + } + #endif + size_t cacheSize = tempSize.empty() ? NGRAPH_EP_LRU_CACHE_DEFAULT_SIZE : std::stoi(tempSize); + + // Not in cache + if (ng_exe_map_.find(uniq_input_shape) == ng_exe_map_.end()) { + // Check if full + if (keyCache.size() == cacheSize) { + // Delete least recently used element + std::string last = keyCache.back(); + + // Pop the last elmeent + keyCache.pop_back(); + + // Erase the last element from cache + ng_exe_map_.erase(ng_exe_map_.find(last)); + } + } + + // Found in cache + else { + keyCache.remove(uniq_input_shape); + } + + // update reference + keyCache.push_front(uniq_input_shape); + auto it = ng_exe_map_.insert({uniq_input_shape, nullptr}); //ng_exe with current shape already exists if (!it.second) { @@ -141,11 +181,11 @@ void NGRAPHCustomOp::Initialize(const OrtCustomOpApi* api, OrtKernelContext* con Status NGRAPHCustomOp::Compute(const OrtCustomOpApi* api, OrtKernelContext* context) const { Ort::CustomOpApi ort{*api}; - //TODO: Minimize locked region - std::lock_guard lock(compute_lock_); - // Initialize nGraph function if it is not already initialized. - Initialize(api, context); + { + std::lock_guard lock(compute_lock_); + Initialize(api, context); + } ORT_ENFORCE(ng_curr_exe_ != nullptr); @@ -158,6 +198,7 @@ Status NGRAPHCustomOp::Compute(const OrtCustomOpApi* api, OrtKernelContext* cont for (const auto& ng_param : ng_curr_exe_->get_parameters()) { const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index++); void* input_data = const_cast(ort.GetTensorData(input_tensor)); + std::lock_guard lock(compute_lock_); ng_inputs.emplace_back(ng_backend_->create_tensor(ng_param->get_output_element_type(0), ng_param->get_output_shape(0), input_data)); } } catch (const std::exception& exp) { @@ -177,6 +218,7 @@ Status NGRAPHCustomOp::Compute(const OrtCustomOpApi* api, OrtKernelContext* cont std::vector ort_shape{shape.begin(), shape.end()}; OrtValue* output_tensor = ort.KernelContext_GetOutput(context, output_index++, ort_shape.data(), ort_shape.size()); void* output_data = ort.GetTensorMutableData(output_tensor); + std::lock_guard lock(compute_lock_); ng_outputs.emplace_back(ng_backend_->create_tensor(dtype, shape, output_data)); } } catch (const std::exception& exp) { @@ -187,6 +229,7 @@ Status NGRAPHCustomOp::Compute(const OrtCustomOpApi* api, OrtKernelContext* cont // Run the graph through nGraph. try { + std::lock_guard lock(compute_lock_); if (!ng_curr_exe_->call(ng_outputs, ng_inputs)) return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, name_ + ": Error while executing nGraph computation"); } catch (const std::exception& exp) { diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h index 2765974821..ad9955872d 100644 --- a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h @@ -56,7 +56,8 @@ class NGRAPHCustomOp { key = [3,1,2,3,2,4,5] */ mutable std::unordered_map> ng_exe_map_; - + mutable std::list keyCache; + mutable std::mutex compute_lock_; mutable ONNX_NAMESPACE::ModelProto model_proto_;