From 0cd4a9381c4705071ea1a21026ecbe7faf820094 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Tue, 2 Apr 2019 20:33:45 -0700 Subject: [PATCH] Handle seqence length == 0 for RNN (#749) * Handle seqence length == 0 for RNN * Zero out the output for seq_len=0 * add unit test for rnn seq len equal 0 * minor fix --- .../core/providers/cpu/rnn/deep_cpu_gru.cc | 34 ++++--- .../core/providers/cpu/rnn/deep_cpu_lstm.cc | 38 ++++++-- .../core/providers/cpu/rnn/rnn_helpers.cc | 2 +- .../core/providers/cpu/rnn/rnn_helpers.h | 2 - .../providers/cpu/rnn/deep_cpu_gru_op_test.cc | 60 ++++++++++++- .../cpu/rnn/deep_cpu_lstm_op_test.cc | 90 +++++++++++++++++++ 6 files changed, 203 insertions(+), 23 deletions(-) diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc index 43171bd9b3..5658e3619a 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_gru.cc @@ -313,11 +313,21 @@ Status DeepCpuGruOp::ComputeImpl(OpKernelContext& context) const { // GRU outputs are optional but must be in the same order TensorShape Y_dims{seq_length, num_directions_, batch_size, hidden_size_}; - Tensor* Y = context.Output(/*index*/ 0, Y_dims); // TODO: Adjust for however optional outputs gets implemented + Tensor* Y = context.Output(/*index*/ 0, Y_dims); TensorShape Y_h_dims{num_directions_, batch_size, hidden_size_}; Tensor* Y_h = context.Output(/*index*/ 1, Y_h_dims); + // Reset output and return if max sequence length is 0 + if (sequence_lens != nullptr) { + int32_t max_sequence_length = *std::max_element(sequence_lens->Data(), sequence_lens->Data() + sequence_lens->Shape().Size()); + if (max_sequence_length == 0) { + if (Y != nullptr) std::fill_n(Y->MutableData(), Y_dims.Size(), T{}); + if (Y_h != nullptr) std::fill_n(Y_h->MutableData(), Y_h_dims.Size(), T{}); + return Status::OK(); + } + } + AllocatorPtr alloc; status = context.GetTempSpaceAllocator(&alloc); ORT_RETURN_IF_ERROR(status); @@ -1024,20 +1034,24 @@ void UniDirectionalGru::Compute(const gsl::span& inputs_arg, } } - if (output_sequence) { - // copy last output to final_hidden_state - for (int i = 0; i < batch_size_; i++) { - const int seq_len = sequence_lengths[i]; + // copy last output to final_hidden_state + for (int i = 0; i < batch_size_; i++) { + const int seq_len = sequence_lengths[i]; + if (seq_len == 0) { + auto final_hidden_state_dst = final_hidden_state.begin() + i * hidden_size_; + std::fill_n(final_hidden_state_dst, hidden_size_, T{}); + continue; + } else if (output_sequence) { auto src = outputs.subspan((seq_len - 1) * output_step_length + i * hidden_size_, hidden_size_); auto dest = final_hidden_state.subspan(i * hidden_size_, hidden_size_); gsl::copy(src, dest); } + } - if (direction_ == kReverse) { - ReverseSequence(outputs, original_outputs, - sequence_lengths, seq_length_, - batch_size_, hidden_size_, num_directions); - } + if (output_sequence && direction_ == kReverse) { + ReverseSequence(outputs, original_outputs, + sequence_lengths, seq_length_, + batch_size_, hidden_size_, num_directions); } } diff --git a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc index 4e1ed72292..a523b0fdcf 100644 --- a/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc +++ b/onnxruntime/core/providers/cpu/rnn/deep_cpu_lstm.cc @@ -369,6 +369,17 @@ Status DeepCpuLstmOp::ComputeImpl(OpKernelContext& context) const { TensorShape Y_c_dims{num_directions_, batch_size, hidden_size_}; Tensor* Y_c = context.Output(/*index*/ 2, Y_c_dims); + // Reset output and return if max sequence length is 0 + if (sequence_lens != nullptr) { + int32_t max_sequence_length = *std::max_element(sequence_lens->Data(), sequence_lens->Data() + sequence_lens->Shape().Size()); + if (max_sequence_length == 0) { + if (Y != nullptr) std::fill_n(Y->MutableData(), Y_dims.Size(), T{}); + if (Y_h != nullptr) std::fill_n(Y_h->MutableData(), Y_h_dims.Size(), T{}); + if (Y_c != nullptr) std::fill_n(Y_c->MutableData(), Y_c_dims.Size(), T{}); + return Status::OK(); + } + } + AllocatorPtr alloc; status = context.GetTempSpaceAllocator(&alloc); ORT_RETURN_IF_ERROR(status); @@ -876,6 +887,10 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, gsl::span dst = final_cell_state.subspan(lrow * hidden_size_, hidden_size_); gsl::copy(src, dst); } + if (step == 0 && sequence_lengths[lrow] == 0) { + auto final_cell_state_dst = final_cell_state.begin() + lrow * hidden_size_; + std::fill_n(final_cell_state_dst, hidden_size_, T{}); + } } if (output_sequence) { @@ -945,6 +960,10 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, gsl::copy(batched_internal_memory_prev_.subspan(lrow * hidden_size_, hidden_size_), final_cell_state.subspan(lrow * hidden_size_, hidden_size_)); } + if (step == 0 && sequence_lengths[lrow] == 0) { + auto final_cell_state_dst = final_cell_state.begin() + lrow * hidden_size_; + std::fill_n(final_cell_state_dst, hidden_size_, T{}); + } } if (output_sequence) { @@ -962,19 +981,22 @@ void UniDirectionalLstm::Compute(const gsl::span& inputs_arg, } } - if (output_sequence) { - // copy last output to final_hidden_state - for (int i = 0; i < batch_size_; i++) { - const int seq_len = sequence_lengths[i]; + for (int i = 0; i < batch_size_; i++) { + const int seq_len = sequence_lengths[i]; + if (seq_len == 0) { // zero out final_hidden_state if seq_len == 0 + auto final_hidden_state_dst = final_hidden_state.begin() + i * hidden_size_; + std::fill_n(final_hidden_state_dst, hidden_size_, T{}); + continue; + } else if (output_sequence) { // copy last output to final_hidden_state auto src = outputs.subspan((seq_len - 1) * output_step_length + i * hidden_size_, hidden_size_); auto dest = final_hidden_state.subspan(i * hidden_size_, hidden_size_); gsl::copy(src, dest); } - - if (direction_ == Direction::kReverse) - ReverseSequence(outputs, original_outputs, sequence_lengths, seq_length_, - batch_size_, hidden_size_, num_directions); } + + if (output_sequence && direction_ == Direction::kReverse) + ReverseSequence(outputs, original_outputs, sequence_lengths, seq_length_, + batch_size_, hidden_size_, num_directions); } // #define PREVIOUS_BROKEN_VERSION diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc index 21c5c09356..42481d8b5f 100644 --- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.cc @@ -79,7 +79,7 @@ Status ValidateCommonRnnInputs(const Tensor& X, auto sequence_len_entries = sequence_lens->DataAsSpan(); if (std::any_of(sequence_len_entries.cbegin(), sequence_len_entries.cend(), - [seq_length](int len) { return len <= 0 || len > seq_length; })) { + [seq_length](int len) { return len < 0 || len > seq_length; })) { return ORT_MAKE_STATUS( ONNXRUNTIME, INVALID_ARGUMENT, "Invalid value/s in sequence_lens. All values must be > 0 and < seq_length. seq_length=", seq_length); diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h index f30ea044a7..9d830e51da 100644 --- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h @@ -120,8 +120,6 @@ void ReverseSequence(gsl::span inputs, for (int i = 0; i < batch_size; i++) { int seq_len = sequence_lengths[i]; - if (seq_len == 0) - continue; #ifdef USE_OPENMP // Parallel execute the loop. #pragma omp parallel for diff --git a/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc b/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc index 0e1cba2e71..22852972bc 100644 --- a/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc +++ b/onnxruntime/test/providers/cpu/rnn/deep_cpu_gru_op_test.cc @@ -56,7 +56,7 @@ static void RunGruTest(const std::vector& X_data, test.AddAttribute("linear_before_reset", linear_before_reset); // if clip is a very big number (usually it is default value), don't set the clip if (clip < 999.f) - test.AddAttribute("clip", clip); + test.AddAttribute("clip", clip); std::vector X_dims = {seq_length, batch_size, input_size}; std::vector W_dims = {num_directions, 3 * hidden_size, input_size}; @@ -457,7 +457,7 @@ void DeepCpuGruOpTestContext::RunTest(const std::vector& X, const std::vector* initial_h, const std::vector& expected_Y, const std::vector& expected_Y_h) { - // run with and without output_sequence + //run with and without output_sequence ::onnxruntime::test::RunGruTest(X, gru_input_weights_, gru_recurrent_weights_, expected_Y, expected_Y_h, input_size_, batch_size, hidden_dim_, seq_length, @@ -652,6 +652,62 @@ TEST(GRUTest, ONNXRuntime_TestGRUOpGrowBatchSequenceLength) { ctx.RunTest(X2, batch2, seq_length2, sequence_length2, &initial_h2, expected_Y2, expected_Y_h2); } +TEST(GRUTest, ONNXRuntime_TestGRUOpSequenceLengthWithPartialZero) { + const std::string direction = "bidirectional"; + const std::vector activations = {"sigmoid", "tanh", "sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch_size = 2; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.455351f, -0.276391f, + -0.185934f, -0.269585f, + -0.185934f, -0.269585f}; + std::vector sequence_length = {2, 0}; + std::vector initial_h = {0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f}; + std::vector expected_Y = {-0.03255286f, 0.0774838f, 0.0f, 0.0f, + -0.05469977f, 0.1004222f, 0.0f, 0.0f, + + -0.05556786f, 0.0785508f, 0.0f, 0.0f, + -0.04566499f, 0.04621252f, 0.0f, 0.0f}; + std::vector expected_Y_h = {-0.05556786f, 0.0785508f, + 0.0f, 0.0f, + -0.05469977f, 0.1004222f, + 0.0f, 0.0f}; + + ctx.RunTest(X, batch_size, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + +TEST(GRUTest, ONNXRuntime_TestGRUOpSequenceLengthAllZeros) { + const std::string direction = "forward"; + const std::vector activations = {"sigmoid", "tanh"}; + + DeepCpuGruOpTestContext ctx(direction, activations); + + const int batch = 2; + const int seq_length = 2; + std::vector X = {-0.455351f, -0.276391f, + -0.455351f, -0.276391f, + + -0.185934f, -0.269585f, + 0.0f, 0.0f}; + std::vector sequence_length = {0, 0}; + std::vector initial_h = {0.0f, 0.0f, + 0.0f, 0.0f}; + std::vector expected_Y = {0.0f, 0.0f, + 0.0f, 0.0f, + + 0.0f, 0.0f, + 0.0f, 0.0f}; + + std::vector expected_Y_h = {0.0f, 0.0f, + 0.0f, 0.0f}; + + ctx.RunTest(X, batch, seq_length, sequence_length, &initial_h, expected_Y, expected_Y_h); +} + TEST(GRUTest, ONNXRuntime_TestGRUOpSingleBatchMultipleHiddenThreads) { const std::string direction = "forward"; const std::vector activations = {"sigmoid", "tanh"}; diff --git a/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc b/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc index 153d4ef603..62de9ab143 100644 --- a/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc +++ b/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc @@ -969,5 +969,95 @@ TEST(LSTMTest, ONNXRuntime_TestLSTMOutputWrite) { nullptr, use_bias, use_peepholes); } +TEST(LSTMTest, ONNXRuntime_TestLSTMSequenceLengthAllZeros) { + const int seq_len = 2; + int batch_size = 2; + std::vector activations = {"tanh", "sigmoid", "tanh", "tanh", "sigmoid", "tanh"}; + + bool use_bias = true; + bool use_peepholes = false; + + std::vector X_data = {-0.455351f, -0.776391f, + -0.355351f, -0.576391f, + + -0.185934f, -0.169585f, + -0.285934f, -0.469585f}; + + std::vector sequence_length = {0, 0}; + + std::vector Y_data = {0.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 0.0f, + + 0.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 0.0f}; + + std::vector Y_h_data = {0.0f, 0.0f, + 0.0f, 0.0f, + + 0.0f, 0.0f, + 0.0f, 0.0f}; + + std::vector Y_c_data = {0.0f, 0.0f, + 0.0f, 0.0f, + + 0.0f, 0.0f, + 0.0f, 0.0f}; + + std::string direction = "bidirectional"; + LstmOpContext2x1x2x2 context(direction, activations); + context.RunTest(X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + &sequence_length, use_bias, use_peepholes); +} + +TEST(LSTMTest, ONNXRuntime_TestLSTMSequenceLengthPartialZeros) { + const int seq_len = 2; + int batch_size = 2; + std::vector activations = {"tanh", "sigmoid", "tanh", "tanh", "sigmoid", "tanh"}; + + bool use_bias = true; + bool use_peepholes = false; + + std::vector X_data = {-0.455351f, -0.776391f, + 0.0f, 0.0f, + + -0.185934f, -0.169585f, + 0.0f, 0.0f}; + + std::vector sequence_length = {2, 0}; + + std::vector Y_data = {-0.1269719f, -0.01049645f, + 0.0f, 0.0f, + + -0.12206709f, -0.0051103f, + 0.0f, 0.0f, + + -0.02778835f, 0.00775075f, + 0.0f, 0.0f, + + -0.04350187f, 0.01127771f, + 0.0f, 0.0f}; + + std::vector Y_h_data = {-0.02778835f, 0.00775075f, + 0.0f, 0.0f, + + -0.12206709f, -0.0051103f, + 0.0f, 0.0f}; + + std::vector Y_c_data = {0.14675268f, 0.01759163f, + 0.0f, 0.0f, + + 0.26577898f, -0.01694398f, + 0.0f, 0.0f}; + + std::string direction = "bidirectional"; + LstmOpContext2x1x2x2 context(direction, activations); + context.RunTest(X_data, batch_size, seq_len, nullptr, nullptr, Y_data, Y_h_data, Y_c_data, + &sequence_length, use_bias, use_peepholes); +} + } // namespace test } // namespace onnxruntime