mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
LSTM performance tweak + cleanup (#3868)
* avoid unneeded buffer zeroing; includes cleanup * fix includes
This commit is contained in:
Tracy Sharpe
GitHub
parent
fe5a20f2d1
commit
19e3dc47d7
6 changed files with 24 additions and 56 deletions
|
|
@ -9,6 +9,8 @@
|
|||
|
||||
#include "uni_dir_attn_lstm.h"
|
||||
|
||||
#include <thread>
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
#include "core/framework/op_kernel_context_internal.h"
|
||||
|
||||
// there's no way to use a raw pointer as the copy destination with std::copy_n
|
||||
// (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset
|
||||
|
|
@ -12,17 +11,6 @@
|
|||
|
||||
#include "core/providers/cpu/rnn/deep_cpu_gru.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <future>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/framework/allocator.h"
|
||||
#include "core/framework/tensor.h"
|
||||
|
||||
#include "core/platform/ort_mutex.h"
|
||||
#include "core/platform/threadpool.h"
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
|
||||
#include <limits>
|
||||
|
||||
#include "core/framework/allocator.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/providers/cpu/rnn/rnn_helpers.h"
|
||||
|
||||
|
|
|
|||
|
|
@ -9,15 +9,8 @@
|
|||
#pragma warning(disable : 4996)
|
||||
#endif
|
||||
|
||||
#include "core/framework/op_kernel_context_internal.h"
|
||||
|
||||
#include "core/providers/cpu/rnn/deep_cpu_lstm.h"
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/framework/allocator.h"
|
||||
#include "core/platform/threadpool.h"
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
|
@ -215,7 +208,7 @@ class UniDirectionalLstm {
|
|||
const gsl::span<const T>& bias, const gsl::span<const T>& peephole_weights,
|
||||
const gsl::span<const T>& initial_hidden_state, const gsl::span<const T>& initial_cell_state,
|
||||
const ActivationFuncs::Entry& activation_func_f, const ActivationFuncs::Entry& activation_func_g,
|
||||
const ActivationFuncs::Entry& activation_func_h, float clip, concurrency::ThreadPool* mlas_tp_);
|
||||
const ActivationFuncs::Entry& activation_func_h, float clip, concurrency::ThreadPool* thread_pool);
|
||||
|
||||
void Compute(const gsl::span<const T>& inputs, const gsl::span<const int>& sequence_lengths, int num_directions,
|
||||
const gsl::span<const T>& input_weights, const gsl::span<const T>& recurrent_weights,
|
||||
|
|
@ -267,18 +260,14 @@ class UniDirectionalLstm {
|
|||
gsl::span<T> hidden0_, batched_hidden0_;
|
||||
|
||||
IAllocatorUniquePtr<T> internal_memory_prev_ptr_, batched_internal_memory_prev_ptr_;
|
||||
IAllocatorUniquePtr<T> internal_memory_cur_ptr_, batched_internal_memory_cur_ptr_;
|
||||
IAllocatorUniquePtr<T> batched_internal_memory_clipped_ptr_;
|
||||
gsl::span<T> internal_memory_prev_, batched_internal_memory_prev_;
|
||||
gsl::span<T> internal_memory_cur_, batched_internal_memory_cur_;
|
||||
gsl::span<T> batched_internal_memory_clipped_;
|
||||
|
||||
IAllocatorUniquePtr<T> bias_WRi_ptr_, bias_WRf_ptr_, bias_WRo_ptr_, bias_WRc_ptr_;
|
||||
IAllocatorUniquePtr<T> batched_bias_WRi_ptr_, batched_bias_WRf_ptr_, batched_bias_WRo_ptr_, batched_bias_WRc_ptr_;
|
||||
IAllocatorUniquePtr<T> peephole_i_ptr_, peephole_f_ptr_, peephole_o_ptr_;
|
||||
IAllocatorUniquePtr<T> inputs_reverse_ptr_, outputs_reverse_ptr_;
|
||||
gsl::span<T> bias_WRi_, bias_WRf_, bias_WRo_, bias_WRc_;
|
||||
gsl::span<T> batched_bias_WRi_, batched_bias_WRf_, batched_bias_WRo_, *batched_bias_WRc_;
|
||||
gsl::span<T> inputs_reverse_, outputs_reverse_;
|
||||
|
||||
#if defined(LSTM_NO_PEEPHOLE_COPY)
|
||||
|
|
@ -296,7 +285,7 @@ class UniDirectionalLstm {
|
|||
ActivationInfo<deepcpu::ActivationFuncPtr> activation_g_;
|
||||
ActivationInfo<deepcpu::LstmMergeGatesFuncPtr> activation_h_;
|
||||
|
||||
concurrency::ThreadPool* mlas_tp_;
|
||||
concurrency::ThreadPool* thread_pool_;
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
|
@ -328,7 +317,7 @@ Status DeepCpuLstmOp::Compute(OpKernelContext* context) const {
|
|||
|
||||
template <typename T>
|
||||
Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const {
|
||||
concurrency::ThreadPool* mlas_thread_pool = context.GetOperatorThreadPool();
|
||||
concurrency::ThreadPool* thread_pool = context.GetOperatorThreadPool();
|
||||
|
||||
auto& logger = context.Logger();
|
||||
|
||||
|
|
@ -461,12 +450,12 @@ Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const {
|
|||
detail::UniDirectionalLstm<T> fw(alloc, logger, seq_length, batch_size, input_size, hidden_size_,
|
||||
Direction::kForward, input_forget_, bias_1, peephole_weights_1, initial_hidden_1,
|
||||
initial_cell_1, activation_funcs_.Entries()[0], activation_funcs_.Entries()[1],
|
||||
activation_funcs_.Entries()[2], clip_, mlas_thread_pool);
|
||||
activation_funcs_.Entries()[2], clip_, thread_pool);
|
||||
|
||||
detail::UniDirectionalLstm<T> bw(alloc, logger, seq_length, batch_size, input_size, hidden_size_,
|
||||
Direction::kReverse, input_forget_, bias_2, peephole_weights_2, initial_hidden_2,
|
||||
initial_cell_2, activation_funcs_.Entries()[3], activation_funcs_.Entries()[4],
|
||||
activation_funcs_.Entries()[5], clip_, mlas_thread_pool);
|
||||
activation_funcs_.Entries()[5], clip_, thread_pool);
|
||||
|
||||
fw.Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1,
|
||||
hidden_output_1, last_cell_1);
|
||||
|
|
@ -476,7 +465,7 @@ Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const {
|
|||
detail::UniDirectionalLstm<T> fw(alloc, logger, seq_length, batch_size, input_size, hidden_size_, direction_,
|
||||
input_forget_, bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1,
|
||||
activation_funcs_.Entries()[0], activation_funcs_.Entries()[1],
|
||||
activation_funcs_.Entries()[2], clip_, mlas_thread_pool);
|
||||
activation_funcs_.Entries()[2], clip_, thread_pool);
|
||||
|
||||
fw.Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1,
|
||||
hidden_output_1, last_cell_1);
|
||||
|
|
@ -535,7 +524,7 @@ UniDirectionalLstm<T>::UniDirectionalLstm(
|
|||
const gsl::span<const T>& bias, const gsl::span<const T>& peephole_weights,
|
||||
const gsl::span<const T>& initial_hidden_state, const gsl::span<const T>& initial_cell_state,
|
||||
const ActivationFuncs::Entry& activation_func_f, const ActivationFuncs::Entry& activation_func_g,
|
||||
const ActivationFuncs::Entry& activation_func_h, const float clip, concurrency::ThreadPool* mlas_tp)
|
||||
const ActivationFuncs::Entry& activation_func_h, const float clip, concurrency::ThreadPool* thread_pool)
|
||||
: allocator_(allocator),
|
||||
logger_(logger),
|
||||
seq_length_(seq_length),
|
||||
|
|
@ -547,7 +536,7 @@ UniDirectionalLstm<T>::UniDirectionalLstm(
|
|||
clip_(clip),
|
||||
use_bias_(!bias.empty()),
|
||||
use_peepholes_(!peephole_weights.empty()),
|
||||
mlas_tp_(mlas_tp) {
|
||||
thread_pool_(thread_pool) {
|
||||
activation_f_ = {deepcpu::ActivationFuncByName(activation_func_f.name), activation_func_f.alpha,
|
||||
activation_func_f.beta};
|
||||
|
||||
|
|
@ -563,29 +552,26 @@ UniDirectionalLstm<T>::UniDirectionalLstm(
|
|||
AllocateBuffers();
|
||||
InitializeBuffers(initial_hidden_state, initial_cell_state);
|
||||
|
||||
if (!peephole_weights.empty())
|
||||
if (use_peepholes_)
|
||||
LoadPeepholeWeights(peephole_weights);
|
||||
if (!bias.empty())
|
||||
if (use_bias_)
|
||||
LoadBias(bias);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void UniDirectionalLstm<T>::AllocateBuffers() {
|
||||
// allocate and fill with 0's.
|
||||
// allocate and fill with zeroes
|
||||
const bool fill = true;
|
||||
hidden0_ = Allocate(allocator_, hidden_size_, hidden0_ptr_, fill);
|
||||
internal_memory_prev_ = Allocate(allocator_, hidden_size_, internal_memory_prev_ptr_, fill);
|
||||
internal_memory_cur_ = Allocate(allocator_, hidden_size_, internal_memory_cur_ptr_, fill);
|
||||
batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_, fill);
|
||||
batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_);
|
||||
|
||||
batched_internal_memory_prev_ =
|
||||
Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_prev_ptr_, fill);
|
||||
batched_internal_memory_cur_ =
|
||||
Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_cur_ptr_, fill);
|
||||
Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_prev_ptr_);
|
||||
batched_internal_memory_clipped_ =
|
||||
Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_clipped_ptr_, fill);
|
||||
|
||||
output_iofc_ = Allocate(allocator_, hidden_size_ * 4 * batch_size_ * seq_length_, output_iofc_ptr_, fill);
|
||||
output_iofc_ = Allocate(allocator_, hidden_size_ * 4 * batch_size_ * seq_length_, output_iofc_ptr_);
|
||||
|
||||
if (use_bias_) {
|
||||
bias_WRi_ = Allocate(allocator_, hidden_size_, bias_WRi_ptr_);
|
||||
|
|
@ -728,7 +714,7 @@ void UniDirectionalLstm<T>::Compute(const gsl::span<const T>& inputs_arg,
|
|||
const bool output_sequence = !outputs.empty();
|
||||
|
||||
if (direction_ == kReverse) {
|
||||
ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1, mlas_tp_);
|
||||
ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1, thread_pool_);
|
||||
inputs = inputs_reverse_;
|
||||
|
||||
if (output_sequence)
|
||||
|
|
@ -752,7 +738,7 @@ void UniDirectionalLstm<T>::Compute(const gsl::span<const T>& inputs_arg,
|
|||
// apply the weights to all the inputs and save to output_IOFC
|
||||
ComputeGemm(total_rows, hidden_size_x4, input_size_, alpha, inputs.cbegin(), inputs.cend(), input_size_,
|
||||
input_weights.cbegin(), input_weights.cend(), // W[iofc]
|
||||
input_size_, beta, output_iofc_.begin(), output_iofc_.end(), hidden_size_x4, mlas_tp_);
|
||||
input_size_, beta, output_iofc_.begin(), output_iofc_.end(), hidden_size_x4, thread_pool_);
|
||||
|
||||
DumpMatrix("Xt*(W[iofc]^T)", output_iofc_.data(), total_rows, hidden_size_x4);
|
||||
|
||||
|
|
@ -852,7 +838,7 @@ void UniDirectionalLstm<T>::Compute(const gsl::span<const T>& inputs_arg,
|
|||
// Approximate worst case cost of hidden_gemm_and_activations.
|
||||
double gemm_cost = fused_hidden_rows * hidden_size_x4 * hidden_size_;
|
||||
double cost = max_sequence_length * (gemm_cost + fused_hidden_rows);
|
||||
ExecuteLambdaInParallel(hidden_gemm_and_activations, batch_size_, fused_hidden_rows, cost, mlas_tp_);
|
||||
ExecuteLambdaInParallel(hidden_gemm_and_activations, batch_size_, fused_hidden_rows, cost, thread_pool_);
|
||||
|
||||
} else {
|
||||
span_T_const_iter previous_state_end = batched_hidden_state_one_step.cend();
|
||||
|
|
@ -878,7 +864,7 @@ void UniDirectionalLstm<T>::Compute(const gsl::span<const T>& inputs_arg,
|
|||
ComputeGemm(batch_size_, hidden_size_x4, hidden_size_, alpha, previous_state, previous_state_end, // Ht-1
|
||||
hidden_size_, recurrent_weights.cbegin(), recurrent_weights.cend(), // R[iofc]
|
||||
hidden_size_, beta, step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T)
|
||||
hidden_size_x4, mlas_tp_);
|
||||
hidden_size_x4, thread_pool_);
|
||||
|
||||
span_T_iter batched_output;
|
||||
span_T_iter batched_output_end;
|
||||
|
|
@ -953,7 +939,7 @@ void UniDirectionalLstm<T>::Compute(const gsl::span<const T>& inputs_arg,
|
|||
|
||||
if (output_sequence && direction_ == Direction::kReverse)
|
||||
ReverseSequence<T>(outputs, original_outputs, sequence_lengths, seq_length_, batch_size_, hidden_size_,
|
||||
num_directions, mlas_tp_);
|
||||
num_directions, thread_pool_);
|
||||
}
|
||||
|
||||
// #define PREVIOUS_BROKEN_VERSION
|
||||
|
|
@ -1083,7 +1069,7 @@ void UniDirectionalLstm<T>::GateComputations(
|
|||
|
||||
template <typename T>
|
||||
void UniDirectionalLstm<T>::SetNumThreads() {
|
||||
int threads = concurrency::ThreadPool::NumThreads(mlas_tp_);
|
||||
int threads = concurrency::ThreadPool::NumThreads(thread_pool_);
|
||||
|
||||
if (threads < 1)
|
||||
threads = 1;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/providers/cpu/rnn/rnn_helpers.h"
|
||||
#include "core/platform/threadpool.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
|
|||
|
|
@ -7,14 +7,6 @@
|
|||
#pragma warning(disable : 4267)
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "gsl/gsl"
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
#include "core/framework/allocator.h"
|
||||
|
|
@ -23,6 +15,8 @@
|
|||
|
||||
#include "core/platform/threadpool.h"
|
||||
|
||||
#include "gsl/gsl"
|
||||
|
||||
namespace onnxruntime {
|
||||
class Tensor;
|
||||
class OpKernelContext;
|
||||
|
|
|
|||
Loading…
Reference in a new issue