diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc index 871fd66252..c52aacdf8b 100644 --- a/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc +++ b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc @@ -9,6 +9,8 @@ #include "uni_dir_attn_lstm.h" +#include + #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc index e030a1ab5f..49ede2c72a 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc @@ -1,6 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/op_kernel_context_internal.h" // there's no way to use a raw pointer as the copy destination with std::copy_n // (which gsl::copy uses with span::data() which returns a raw pointer) with the 14.11 toolset @@ -12,17 +11,6 @@ #include "core/providers/cpu/rnn/deep_cpu_gru.h" -#include -#include -#include - -#include "core/common/logging/logging.h" -#include "core/framework/allocator.h" -#include "core/framework/tensor.h" - -#include "core/platform/ort_mutex.h" -#include "core/platform/threadpool.h" - #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h index 3f6294eb84..0a34697b30 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.h @@ -5,7 +5,6 @@ #include -#include "core/framework/allocator.h" #include "core/framework/op_kernel.h" #include "core/providers/cpu/rnn/rnn_helpers.h" diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc index 9d4e7c2ec2..97dc76d730 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc @@ -9,15 +9,8 @@ #pragma warning(disable : 4996) #endif -#include "core/framework/op_kernel_context_internal.h" - #include "core/providers/cpu/rnn/deep_cpu_lstm.h" -#include "core/common/common.h" -#include "core/common/logging/logging.h" -#include "core/framework/allocator.h" -#include "core/platform/threadpool.h" - #ifdef _MSC_VER #pragma warning(pop) #endif @@ -215,7 +208,7 @@ class UniDirectionalLstm { const gsl::span& bias, const gsl::span& peephole_weights, const gsl::span& initial_hidden_state, const gsl::span& initial_cell_state, const ActivationFuncs::Entry& activation_func_f, const ActivationFuncs::Entry& activation_func_g, - const ActivationFuncs::Entry& activation_func_h, float clip, concurrency::ThreadPool* mlas_tp_); + const ActivationFuncs::Entry& activation_func_h, float clip, concurrency::ThreadPool* thread_pool); void Compute(const gsl::span& inputs, const gsl::span& sequence_lengths, int num_directions, const gsl::span& input_weights, const gsl::span& recurrent_weights, @@ -267,18 +260,14 @@ class UniDirectionalLstm { gsl::span hidden0_, batched_hidden0_; IAllocatorUniquePtr internal_memory_prev_ptr_, batched_internal_memory_prev_ptr_; - IAllocatorUniquePtr internal_memory_cur_ptr_, batched_internal_memory_cur_ptr_; IAllocatorUniquePtr batched_internal_memory_clipped_ptr_; gsl::span internal_memory_prev_, batched_internal_memory_prev_; - gsl::span internal_memory_cur_, batched_internal_memory_cur_; gsl::span batched_internal_memory_clipped_; IAllocatorUniquePtr bias_WRi_ptr_, bias_WRf_ptr_, bias_WRo_ptr_, bias_WRc_ptr_; - IAllocatorUniquePtr batched_bias_WRi_ptr_, batched_bias_WRf_ptr_, batched_bias_WRo_ptr_, batched_bias_WRc_ptr_; IAllocatorUniquePtr peephole_i_ptr_, peephole_f_ptr_, peephole_o_ptr_; IAllocatorUniquePtr inputs_reverse_ptr_, outputs_reverse_ptr_; gsl::span bias_WRi_, bias_WRf_, bias_WRo_, bias_WRc_; - gsl::span batched_bias_WRi_, batched_bias_WRf_, batched_bias_WRo_, *batched_bias_WRc_; gsl::span inputs_reverse_, outputs_reverse_; #if defined(LSTM_NO_PEEPHOLE_COPY) @@ -296,7 +285,7 @@ class UniDirectionalLstm { ActivationInfo activation_g_; ActivationInfo activation_h_; - concurrency::ThreadPool* mlas_tp_; + concurrency::ThreadPool* thread_pool_; }; } // namespace detail @@ -328,7 +317,7 @@ Status DeepCpuLstmOp::Compute(OpKernelContext* context) const { template Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const { - concurrency::ThreadPool* mlas_thread_pool = context.GetOperatorThreadPool(); + concurrency::ThreadPool* thread_pool = context.GetOperatorThreadPool(); auto& logger = context.Logger(); @@ -461,12 +450,12 @@ Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const { detail::UniDirectionalLstm fw(alloc, logger, seq_length, batch_size, input_size, hidden_size_, Direction::kForward, input_forget_, bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1, activation_funcs_.Entries()[0], activation_funcs_.Entries()[1], - activation_funcs_.Entries()[2], clip_, mlas_thread_pool); + activation_funcs_.Entries()[2], clip_, thread_pool); detail::UniDirectionalLstm bw(alloc, logger, seq_length, batch_size, input_size, hidden_size_, Direction::kReverse, input_forget_, bias_2, peephole_weights_2, initial_hidden_2, initial_cell_2, activation_funcs_.Entries()[3], activation_funcs_.Entries()[4], - activation_funcs_.Entries()[5], clip_, mlas_thread_pool); + activation_funcs_.Entries()[5], clip_, thread_pool); fw.Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1, last_cell_1); @@ -476,7 +465,7 @@ Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const { detail::UniDirectionalLstm fw(alloc, logger, seq_length, batch_size, input_size, hidden_size_, direction_, input_forget_, bias_1, peephole_weights_1, initial_hidden_1, initial_cell_1, activation_funcs_.Entries()[0], activation_funcs_.Entries()[1], - activation_funcs_.Entries()[2], clip_, mlas_thread_pool); + activation_funcs_.Entries()[2], clip_, thread_pool); fw.Compute(input, sequence_lens_span, num_directions_, input_weights_1, recurrent_weights_1, output_1, hidden_output_1, last_cell_1); @@ -535,7 +524,7 @@ UniDirectionalLstm::UniDirectionalLstm( const gsl::span& bias, const gsl::span& peephole_weights, const gsl::span& initial_hidden_state, const gsl::span& initial_cell_state, const ActivationFuncs::Entry& activation_func_f, const ActivationFuncs::Entry& activation_func_g, - const ActivationFuncs::Entry& activation_func_h, const float clip, concurrency::ThreadPool* mlas_tp) + const ActivationFuncs::Entry& activation_func_h, const float clip, concurrency::ThreadPool* thread_pool) : allocator_(allocator), logger_(logger), seq_length_(seq_length), @@ -547,7 +536,7 @@ UniDirectionalLstm::UniDirectionalLstm( clip_(clip), use_bias_(!bias.empty()), use_peepholes_(!peephole_weights.empty()), - mlas_tp_(mlas_tp) { + thread_pool_(thread_pool) { activation_f_ = {deepcpu::ActivationFuncByName(activation_func_f.name), activation_func_f.alpha, activation_func_f.beta}; @@ -563,29 +552,26 @@ UniDirectionalLstm::UniDirectionalLstm( AllocateBuffers(); InitializeBuffers(initial_hidden_state, initial_cell_state); - if (!peephole_weights.empty()) + if (use_peepholes_) LoadPeepholeWeights(peephole_weights); - if (!bias.empty()) + if (use_bias_) LoadBias(bias); } template void UniDirectionalLstm::AllocateBuffers() { - // allocate and fill with 0's. + // allocate and fill with zeroes const bool fill = true; hidden0_ = Allocate(allocator_, hidden_size_, hidden0_ptr_, fill); internal_memory_prev_ = Allocate(allocator_, hidden_size_, internal_memory_prev_ptr_, fill); - internal_memory_cur_ = Allocate(allocator_, hidden_size_, internal_memory_cur_ptr_, fill); - batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_, fill); + batched_hidden0_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_hidden0_ptr_); batched_internal_memory_prev_ = - Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_prev_ptr_, fill); - batched_internal_memory_cur_ = - Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_cur_ptr_, fill); + Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_prev_ptr_); batched_internal_memory_clipped_ = Allocate(allocator_, batch_size_ * hidden_size_, batched_internal_memory_clipped_ptr_, fill); - output_iofc_ = Allocate(allocator_, hidden_size_ * 4 * batch_size_ * seq_length_, output_iofc_ptr_, fill); + output_iofc_ = Allocate(allocator_, hidden_size_ * 4 * batch_size_ * seq_length_, output_iofc_ptr_); if (use_bias_) { bias_WRi_ = Allocate(allocator_, hidden_size_, bias_WRi_ptr_); @@ -728,7 +714,7 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, const bool output_sequence = !outputs.empty(); if (direction_ == kReverse) { - ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1, mlas_tp_); + ReverseSequence(inputs, inputs_reverse_, sequence_lengths, seq_length_, batch_size_, input_size_, 1, thread_pool_); inputs = inputs_reverse_; if (output_sequence) @@ -752,7 +738,7 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, // apply the weights to all the inputs and save to output_IOFC ComputeGemm(total_rows, hidden_size_x4, input_size_, alpha, inputs.cbegin(), inputs.cend(), input_size_, input_weights.cbegin(), input_weights.cend(), // W[iofc] - input_size_, beta, output_iofc_.begin(), output_iofc_.end(), hidden_size_x4, mlas_tp_); + input_size_, beta, output_iofc_.begin(), output_iofc_.end(), hidden_size_x4, thread_pool_); DumpMatrix("Xt*(W[iofc]^T)", output_iofc_.data(), total_rows, hidden_size_x4); @@ -852,7 +838,7 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, // Approximate worst case cost of hidden_gemm_and_activations. double gemm_cost = fused_hidden_rows * hidden_size_x4 * hidden_size_; double cost = max_sequence_length * (gemm_cost + fused_hidden_rows); - ExecuteLambdaInParallel(hidden_gemm_and_activations, batch_size_, fused_hidden_rows, cost, mlas_tp_); + ExecuteLambdaInParallel(hidden_gemm_and_activations, batch_size_, fused_hidden_rows, cost, thread_pool_); } else { span_T_const_iter previous_state_end = batched_hidden_state_one_step.cend(); @@ -878,7 +864,7 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, ComputeGemm(batch_size_, hidden_size_x4, hidden_size_, alpha, previous_state, previous_state_end, // Ht-1 hidden_size_, recurrent_weights.cbegin(), recurrent_weights.cend(), // R[iofc] hidden_size_, beta, step_out_IOFC, output_iofc_.end(), // input contains Xt*(W[iofc]^T) - hidden_size_x4, mlas_tp_); + hidden_size_x4, thread_pool_); span_T_iter batched_output; span_T_iter batched_output_end; @@ -953,7 +939,7 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, if (output_sequence && direction_ == Direction::kReverse) ReverseSequence(outputs, original_outputs, sequence_lengths, seq_length_, batch_size_, hidden_size_, - num_directions, mlas_tp_); + num_directions, thread_pool_); } // #define PREVIOUS_BROKEN_VERSION @@ -1083,7 +1069,7 @@ void UniDirectionalLstm::GateComputations( template void UniDirectionalLstm::SetNumThreads() { - int threads = concurrency::ThreadPool::NumThreads(mlas_tp_); + int threads = concurrency::ThreadPool::NumThreads(thread_pool_); if (threads < 1) threads = 1; diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h index b6fa3ea404..a3adeb7cd5 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.h @@ -7,7 +7,6 @@ #include "core/framework/op_kernel.h" #include "core/providers/cpu/rnn/rnn_helpers.h" -#include "core/platform/threadpool.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h index 4b7078ae7c..5c221a6ccd 100644 --- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h @@ -7,14 +7,6 @@ #pragma warning(disable : 4267) #endif -#include -#include -#include -#include -#include - -#include "gsl/gsl" - #include "core/common/common.h" #include "core/common/logging/logging.h" #include "core/framework/allocator.h" @@ -23,6 +15,8 @@ #include "core/platform/threadpool.h" +#include "gsl/gsl" + namespace onnxruntime { class Tensor; class OpKernelContext;