saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-08 17:17:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use sequences to create initial feeds for decoder subgraph (#13719)

Browse source 
Use sequences to create initial feeds for decoder subgraph instead of
beam_next_tokens

### Description
For TuLG models exporting of decoder is different from bart model.
Passing beam_next_tokens to the decoder while ort inferencing generated
incorrect result from pytorch inference.
This change will use sequences as inputs for the first iteration as well


### Motivation and Context
Pytorch and ORT inference for TuLG models was incorrect, keeping pytorch
as correct result we modified ort to match the result.
This commit is contained in:
apsonawane 2022-11-22 18:00:58 -08:00 committed by GitHub
parent fb85b31fac
commit 7857f59d2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 33 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_t5.h
View file

@ -225,7 +225,10 @@ Status BeamSearchT5<T>::Execute(const FeedsFetchesManager& encoder_feeds_fetches
this->expand_buffer_float_func_,
this->expand_buffer_float16_func_,
parameters->num_beams,
this->cuda_stream_));
this->cuda_stream_,
decoder_subgraph_.UseSequenceAsInputIds(),
current_length,
cpu_state.sequences));
}
// TODO(tianleiwu): allocate fetches. use ping-pong buffers for past state.

31
onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.cc
View file

@ -10,6 +10,7 @@
#include "contrib_ops/cpu/transformers/subgraph_t5_decoder.h"
#include "contrib_ops/cpu/transformers/dump_tensor.h"
#include "contrib_ops/cpu/transformers/generation_device_helper.h"
#include "contrib_ops/cpu/transformers/sequences.h"
namespace onnxruntime {
namespace contrib {
@ -132,7 +133,10 @@ Status T5DecoderSubgraph::CreateInitialFeeds(
const GenerationDeviceHelper::ExpandBufferFunc<float>& expand_buffer_float_func,
const GenerationDeviceHelper::ExpandBufferFunc<MLFloat16>& expand_buffer_float16_func,
int num_beam,
void* stream) {
void* stream,
bool use_sequence_as_input_ids,
int cur_len,
transformers::Sequences& sequences) {
ORT_ENFORCE(session_state_ != nullptr, "Setup must be called before CreateInitialFeeds");
// Allocate subgraph inputs from same device as inputs of encoder subgraph.
@ -140,15 +144,28 @@ Status T5DecoderSubgraph::CreateInitialFeeds(
// Copy beam next tokens in CPU to input_ids in provider device (CPU for CPU EP, or GPU for CUDA EP).
int batch_beam_size = static_cast<int>(beam_next_tokens.size());
int64_t dims[] = {batch_beam_size, 1};
int sequence_length = !use_sequence_as_input_ids ? 1 : cur_len;
int64_t dims[] = {batch_beam_size, sequence_length};
TensorShape input_ids_shape(&dims[0], 2);
OrtValue input_ids;
Tensor::InitOrtValue(DataTypeImpl::GetType<int32_t>(), input_ids_shape, allocator, input_ids);
ORT_RETURN_IF_ERROR(device_copy_int32_func(
input_ids.GetMutable<Tensor>()->MutableDataAsSpan<int32_t>(),
beam_next_tokens,
stream,
DeviceCopyDirection::hostToDevice));
int32_t* input_ids_data = input_ids.GetMutable<Tensor>()->MutableData<int32_t>();
if (!use_sequence_as_input_ids_){
ORT_RETURN_IF_ERROR(device_copy_int32_func(
input_ids.GetMutable<Tensor>()->MutableDataAsSpan<int32_t>(),
beam_next_tokens,
stream,
DeviceCopyDirection::hostToDevice));
}else{
for (int i = 0; i < batch_beam_size; i++) {
gsl::span<const int32_t> sequence = sequences.GetSequence(i);
const int32_t* sequence_data = sequence.data();
for (int j = 0; j < cur_len; j++) {
input_ids_data[i * cur_len + j] = sequence_data[j];
}
}
}
// The ordering is the same as used in Setup.
decoder_feeds.reserve(static_cast<size_t>(num_subgraph_inputs) + static_cast<size_t>(num_implicit_inputs));

6
onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.h
View file

@ -4,6 +4,7 @@
#pragma once
#include "contrib_ops/cpu/transformers/subgraph_base.h"
#include "contrib_ops/cpu/transformers/sequences.h"
namespace onnxruntime {
namespace contrib {
@ -33,7 +34,10 @@ class T5DecoderSubgraph : public Subgraph {
const GenerationDeviceHelper::ExpandBufferFunc<float>& expand_buffer_float_func,
const GenerationDeviceHelper::ExpandBufferFunc<MLFloat16>& expand_buffer_float16_func,
int num_beam,
void* stream);
void* stream,
bool use_sequence_as_input_ids,
int cur_len,
transformers::Sequences& sequences);
Status Validate(const std::vector<const NodeArg*>& subgraph_inputs,
const std::vector<const NodeArg*>& subgraph_outputs) override;