mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-24 19:43:35 +00:00
Fix bug in handling of an initializer that provides a graph output. (#3912)
* Outputs from model execution should always be returned in a newly allocated buffer or an pre-allocated buffer provided in fetches. When an initializer is providing a graph output (e.g. constant folding may result in this) we were returning an OrtValue that pointed to the initializer and not a separately allocated buffer with a copy. This was wrong as: - value wasn't returned in a pre-allocated fetch so whilst the value returned was correct, it was returned in the wrong place - user could alter the data in the initializer via the returned value * Add unit test with and without pre-allocated fetch. * Add some extra info around why we're handling this special case.
This commit is contained in:
Scott McKay
GitHub
parent
6f729b100f
commit
2fed37c8eb
6 changed files with 169 additions and 62 deletions
|
|
@ -20,19 +20,14 @@ using namespace onnxruntime::common;
|
|||
|
||||
namespace onnxruntime {
|
||||
|
||||
IExecutionFrame::IExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const std::vector<OrtValue>& feeds,
|
||||
const std::unordered_map<int, OrtValue>& initializers,
|
||||
const std::vector<int>& fetch_mlvalue_idxs, const std::vector<OrtValue>& fetches,
|
||||
const OrtValueNameIdxMap& ort_value_idx_map, const NodeIndexInfo& node_index_info)
|
||||
IExecutionFrame::IExecutionFrame(const OrtValueNameIdxMap& ort_value_idx_map,
|
||||
const NodeIndexInfo& node_index_info,
|
||||
const std::vector<int>& fetch_mlvalue_idxs)
|
||||
: node_index_info_(node_index_info),
|
||||
all_values_size_(static_cast<size_t>(ort_value_idx_map.MaxIdx()) + 1),
|
||||
fetch_mlvalue_idxs_(fetch_mlvalue_idxs) {
|
||||
ORT_ENFORCE(feeds.size() == feed_mlvalue_idxs.size());
|
||||
ORT_ENFORCE(fetches.empty() || fetches.size() == fetch_mlvalue_idxs_.size());
|
||||
ORT_ENFORCE(node_index_info_.GetMaxMLValueIdx() == ort_value_idx_map.MaxIdx(),
|
||||
"node_index_info and ort_value_idx_map are out of sync and cannot be used");
|
||||
|
||||
Init(feed_mlvalue_idxs, feeds, initializers, fetches);
|
||||
}
|
||||
|
||||
IExecutionFrame::~IExecutionFrame() = default;
|
||||
|
|
@ -109,6 +104,9 @@ int IExecutionFrame::GetNodeIdxToMLValueIdx(int index) const {
|
|||
void IExecutionFrame::Init(const std::vector<int>& feed_mlvalue_idxs, const std::vector<OrtValue>& feeds,
|
||||
const std::unordered_map<int, OrtValue>& initializers,
|
||||
const std::vector<OrtValue>& fetches) {
|
||||
ORT_ENFORCE(feeds.size() == feed_mlvalue_idxs.size());
|
||||
ORT_ENFORCE(fetches.empty() || fetches.size() == fetch_mlvalue_idxs_.size());
|
||||
|
||||
// 1. resize the all_value_ vector
|
||||
all_values_.resize(all_values_size_);
|
||||
|
||||
|
|
@ -123,15 +121,44 @@ void IExecutionFrame::Init(const std::vector<int>& feed_mlvalue_idxs, const std:
|
|||
}
|
||||
|
||||
// 3. handle the weights.
|
||||
// We do this after the fetches to handle an edge case (possibly dubious) where a Constant is an output.
|
||||
// The Constant gets lifted to an initializer so there's no Node producing the value as an output during Graph
|
||||
// execution (i.e. Graph execution won't write the value to all_values_).
|
||||
// We do this after the fetches to handle an edge case where an initializer is an output.
|
||||
// e.g. A Constant node gets lifted to an initializer so there's no Node producing the value as an output during
|
||||
// Graph execution (i.e. Graph execution won't write the value to all_values_).
|
||||
// A non-empty fetches vector will overwrite the actual weight in all_values_[ort_value_idx] if we did this earlier.
|
||||
// This makes the ONNX Constant test (onnx\backend\test\data\node\test_constant) happy as that
|
||||
// involves a graph with a single Constant node.
|
||||
for (const auto& entry : initializers) {
|
||||
int ort_value_index = entry.first;
|
||||
all_values_[ort_value_index] = entry.second;
|
||||
|
||||
// if the initializer is an output we need to allocate or use a provided fetch buffer and copy the data
|
||||
// so it can be returned to the caller.
|
||||
//
|
||||
// The alternative to handling this as a special case would be to disallow an initializer providing a graph output.
|
||||
// There's nothing in the ONNX spec that says a graph output must come from a node output though.
|
||||
// If we took that approach we'd need to:
|
||||
// - reject a model with an initializer or Constant node (as we convert those to initializers in Graph::Graph)
|
||||
// that produces a graph output even though it conforms to the ONNX spec
|
||||
// - update optimizers to not convert something to an initializer that is a graph output
|
||||
// (e.g. constant folding)
|
||||
if (IsOutput(ort_value_index)) {
|
||||
const Tensor& src = entry.second.Get<Tensor>(); // all initializers in ONNX are tensors
|
||||
OrtValue& dest = all_values_[ort_value_index];
|
||||
|
||||
if (!dest.IsAllocated()) {
|
||||
// NOTE: This doesn't need to support ExecutionFrame custom allocators as they only come into play
|
||||
// for a subgraph with an output of unknown shape that needs to be accumulated by the control flow node.
|
||||
// If the initializer is providing the output, the shape is known.
|
||||
AllocatorPtr allocator = GetAllocator(src.Location());
|
||||
|
||||
auto p_tensor = onnxruntime::make_unique<Tensor>(src.DataType(), src.Shape(), allocator);
|
||||
auto ml_tensor = DataTypeImpl::GetType<Tensor>();
|
||||
dest.Init(p_tensor.release(), ml_tensor, ml_tensor->GetDeleteFunc());
|
||||
}
|
||||
|
||||
ORT_THROW_IF_ERROR(CopyTensor(src, *dest.GetMutable<Tensor>()));
|
||||
} else {
|
||||
all_values_[ort_value_index] = entry.second;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. handle feed in values. these can override initializer values so must be last
|
||||
|
|
@ -171,11 +198,12 @@ ExecutionFrame::ExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const
|
|||
const std::vector<int>& fetch_mlvalue_idxs, const std::vector<OrtValue>& fetches,
|
||||
const std::unordered_map<size_t, IExecutor::CustomAllocator>& fetch_allocators,
|
||||
const SessionState& session_state)
|
||||
: IExecutionFrame(feed_mlvalue_idxs, feeds, session_state.GetInitializedTensors(), fetch_mlvalue_idxs, fetches,
|
||||
session_state.GetOrtValueNameIdxMap(), session_state.GetNodeIndexInfo()),
|
||||
: IExecutionFrame(session_state.GetOrtValueNameIdxMap(), session_state.GetNodeIndexInfo(), fetch_mlvalue_idxs),
|
||||
session_state_(session_state),
|
||||
mem_patterns_(nullptr),
|
||||
planner_(nullptr) {
|
||||
Init(feed_mlvalue_idxs, feeds, session_state.GetInitializedTensors(), fetches);
|
||||
|
||||
// map the custom allocators to ort_value_idx entries
|
||||
if (!fetch_allocators.empty()) {
|
||||
for (size_t idx = 0, end = fetch_mlvalue_idxs.size(); idx < end; ++idx) {
|
||||
|
|
@ -232,6 +260,10 @@ ExecutionFrame::ExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const
|
|||
|
||||
ExecutionFrame::~ExecutionFrame() = default;
|
||||
|
||||
Status ExecutionFrame::CopyTensor(const Tensor& src, Tensor& dest) const {
|
||||
return session_state_.GetDataTransferMgr().CopyTensor(src, dest);
|
||||
}
|
||||
|
||||
Status ExecutionFrame::AllocateMLValueTensorSelfOwnBuffer(OrtValue& ort_value, int ort_value_index,
|
||||
MLDataType element_type, const OrtMemoryInfo& location,
|
||||
const TensorShape& shape, bool create_fence) {
|
||||
|
|
@ -343,7 +375,7 @@ Status ExecutionFrame::AllocateMLValueTensorPreAllocateBuffer(OrtValue& ort_valu
|
|||
|
||||
// be generous and use the buffer if it's large enough. log a warning though as it indicates a bad model
|
||||
if (buffer_num_elements >= required_num_elements) {
|
||||
// View Operator is reusing the buffer bigger than the required size.
|
||||
// View Operator is reusing the buffer bigger than the required size.
|
||||
// Disabling warning message for now. The op is in the process of being deprecated.
|
||||
#ifndef ENABLE_TRAINING
|
||||
LOGS(session_state_.Logger(), WARNING) << message;
|
||||
|
|
@ -455,13 +487,13 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(OrtValue& ort_value, int ort_
|
|||
}
|
||||
case AllocKind::kReuse: {
|
||||
int reuse_mlvalue_index = per_alloc_plan.reused_buffer;
|
||||
|
||||
|
||||
// In case OrtRunOptions.only_execute_path_to_fetches == true, it is possible that 'reuse_value'
|
||||
// is not allocated (its upstream op is not executed due to the option).
|
||||
// In this case we need to allocate 'reuse_value' and then let 'ort_value' to reuse it.
|
||||
// In this case we need to allocate 'reuse_value' and then let 'ort_value' to reuse it.
|
||||
OrtValue& reuse_value = GetMutableMLValue(reuse_mlvalue_index);
|
||||
if (!reuse_value.IsAllocated()) {
|
||||
ORT_RETURN_IF_ERROR(AllocateAsPerAllocationPlan(reuse_value, reuse_mlvalue_index, shape, nnz));
|
||||
ORT_RETURN_IF_ERROR(AllocateAsPerAllocationPlan(reuse_value, reuse_mlvalue_index, shape, nnz));
|
||||
}
|
||||
ORT_RETURN_IF_ERROR(AllocateMLValueTensorPreAllocateBuffer(
|
||||
ort_value, reuse_mlvalue_index, ml_data_type, alloc_info, *shape, per_alloc_plan.create_fence_if_async));
|
||||
|
|
@ -497,7 +529,8 @@ AllocatorPtr ExecutionFrame::GetAllocatorImpl(const OrtMemoryInfo& info) const {
|
|||
|
||||
// This method is not thread safe!
|
||||
// Return S_OK and nullptr if index map to an value that is an unused optional input/output
|
||||
Status ExecutionFrame::CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx, const TensorShape* shape, size_t nnz) {
|
||||
Status ExecutionFrame::CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx,
|
||||
const TensorShape* shape, size_t nnz) {
|
||||
return AllocateAsPerAllocationPlan(ort_value, ort_value_idx, shape, nnz);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,10 +25,15 @@ class NodeIndexInfo;
|
|||
|
||||
class IExecutionFrame {
|
||||
protected:
|
||||
IExecutionFrame(const std::vector<int>& feed_mlvalue_idxs, const std::vector<OrtValue>& feeds,
|
||||
const std::unordered_map<int, OrtValue>& initializers, const std::vector<int>& fetch_mlvalue_idxs,
|
||||
const std::vector<OrtValue>& fetches, const OrtValueNameIdxMap& ort_value_idx_map,
|
||||
const NodeIndexInfo& node_index_info);
|
||||
// Derived class must call Init in its ctor. We need to use some of the virtual methods in Init and those aren't
|
||||
// initialized until the derived class is constructed.
|
||||
IExecutionFrame(const OrtValueNameIdxMap& ort_value_idx_map,
|
||||
const NodeIndexInfo& node_index_info,
|
||||
const std::vector<int>& fetch_mlvalue_idxs);
|
||||
|
||||
void Init(const std::vector<int>& feed_mlvalue_idxs, const std::vector<OrtValue>& feeds,
|
||||
const std::unordered_map<int, OrtValue>& initializers,
|
||||
const std::vector<OrtValue>& fetches);
|
||||
|
||||
public:
|
||||
virtual ~IExecutionFrame();
|
||||
|
|
@ -72,10 +77,6 @@ class IExecutionFrame {
|
|||
private:
|
||||
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(IExecutionFrame);
|
||||
|
||||
void Init(const std::vector<int>& feed_mlvalue_idxs, const std::vector<OrtValue>& feeds,
|
||||
const std::unordered_map<int, OrtValue>& initializers,
|
||||
const std::vector<OrtValue>& fetches);
|
||||
|
||||
const OrtValue& GetMLValue(int ort_value_index) const {
|
||||
ORT_ENFORCE(ort_value_index >= 0 && static_cast<size_t>(ort_value_index) < all_values_size_);
|
||||
return all_values_[ort_value_index];
|
||||
|
|
@ -83,7 +84,10 @@ class IExecutionFrame {
|
|||
|
||||
virtual AllocatorPtr GetAllocatorImpl(const OrtMemoryInfo& info) const = 0;
|
||||
|
||||
virtual Status CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx, const TensorShape* shape, size_t nnz) = 0;
|
||||
virtual Status CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx, const TensorShape* shape,
|
||||
size_t nnz) = 0;
|
||||
|
||||
virtual Status CopyTensor(const Tensor& src, Tensor& dest) const = 0;
|
||||
|
||||
const NodeIndexInfo& node_index_info_;
|
||||
|
||||
|
|
@ -131,6 +135,7 @@ class ExecutionFrame final : public IExecutionFrame {
|
|||
AllocatorPtr GetAllocatorImpl(const OrtMemoryInfo& info) const override;
|
||||
Status ReleaseMLValueImpl(int ort_value_idx) override;
|
||||
Status CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx, const TensorShape* shape, size_t nnz) override;
|
||||
Status CopyTensor(const Tensor& src, Tensor& dest) const override;
|
||||
|
||||
common::Status AllocateAsPerAllocationPlan(OrtValue& ort_value, int ort_value_index, const TensorShape* shape,
|
||||
size_t nnz);
|
||||
|
|
|
|||
|
|
@ -40,12 +40,12 @@ OptimizerExecutionFrame::Info::Info(const std::vector<const Node*>& nodes,
|
|||
size_t cpu_tensor_length;
|
||||
ORT_RETURN_IF_ERROR(utils::GetSizeInBytesFromTensorProto<0>(tensor_proto, &cpu_tensor_length));
|
||||
OrtValue ort_value;
|
||||
const OrtMemoryInfo& info = cpu_execution_provider_->GetAllocator(0, OrtMemTypeDefault)->Info();
|
||||
std::unique_ptr<char[]> data(new char[cpu_tensor_length]);
|
||||
std::unique_ptr<Tensor> p_tensor;
|
||||
OrtCallback d;
|
||||
ORT_RETURN_IF_ERROR(utils::TensorProtoToMLValue(Env::Default(), nullptr, tensor_proto,
|
||||
MemBuffer(data.get(), cpu_tensor_length, info), ort_value, d));
|
||||
MemBuffer(data.get(), cpu_tensor_length, allocator_ptr_->Info()),
|
||||
ort_value, d));
|
||||
|
||||
initializers_[idx] = ort_value;
|
||||
buffer_for_initialized_tensors_[idx] = std::move(data);
|
||||
|
|
@ -83,14 +83,19 @@ std::unique_ptr<const OpKernel> OptimizerExecutionFrame::Info::CreateKernel(cons
|
|||
// For optimizer, probably no need to pass feed_mlvalue_idxs, feeds to initialize IExecutionFrame.
|
||||
// If needed, the parameters of OptimizerExecutionFrame ctor can be changed later.
|
||||
OptimizerExecutionFrame::OptimizerExecutionFrame(const Info& info, const std::vector<int>& fetch_mlvalue_idxs)
|
||||
: IExecutionFrame(std::vector<int>(), std::vector<OrtValue>(), info.GetInitializers(), fetch_mlvalue_idxs,
|
||||
std::vector<OrtValue>(), info.GetMLValueNameIdxMap(), info.GetNodeIndexInfo()),
|
||||
info_(info) {}
|
||||
: IExecutionFrame(info.GetMLValueNameIdxMap(), info.GetNodeIndexInfo(), fetch_mlvalue_idxs),
|
||||
info_(info) {
|
||||
Init(std::vector<int>(), std::vector<OrtValue>(), info.GetInitializers(), std::vector<OrtValue>());
|
||||
}
|
||||
|
||||
AllocatorPtr OptimizerExecutionFrame::GetAllocatorImpl(const OrtMemoryInfo& info) const {
|
||||
return info_.GetAllocator(info);
|
||||
}
|
||||
|
||||
Status OptimizerExecutionFrame::CopyTensor(const Tensor& src, Tensor& dest) const {
|
||||
return info_.GetDataTransferManager().CopyTensor(src, dest);
|
||||
}
|
||||
|
||||
// This method is not thread safe!
|
||||
// Return S_OK and nullptr if index map to an value that is an unused optional input/output
|
||||
Status OptimizerExecutionFrame::CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx,
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ class OptimizerExecutionFrame final : public IExecutionFrame {
|
|||
|
||||
std::unique_ptr<const OpKernel> CreateKernel(const Node* node) const;
|
||||
|
||||
const DataTransferManager& GetDataTransferManager() const { return data_transfer_mgr_; }
|
||||
|
||||
private:
|
||||
// The optimizer is running on CPU execution provider by default.
|
||||
std::unique_ptr<CPUExecutionProvider> cpu_execution_provider_;
|
||||
|
|
@ -83,6 +85,8 @@ class OptimizerExecutionFrame final : public IExecutionFrame {
|
|||
|
||||
Status CreateNodeOutputMLValueImpl(OrtValue& ort_value, int ort_value_idx, const TensorShape* shape, size_t nnz) override;
|
||||
|
||||
Status CopyTensor(const Tensor& src, Tensor& dest) const override;
|
||||
|
||||
const Info& info_;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/common/make_unique.h"
|
||||
#include "core/framework/execution_frame.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/framework/session_state.h"
|
||||
|
|
@ -9,6 +10,7 @@
|
|||
#include "core/session/inference_session.h"
|
||||
#include "test_utils.h"
|
||||
#include "test/test_environment.h"
|
||||
#include "test/framework/TestAllocatorManager.h"
|
||||
#include "asserts.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "gmock/gmock.h"
|
||||
|
|
@ -20,17 +22,6 @@ namespace onnxruntime {
|
|||
namespace test {
|
||||
typedef std::vector<onnxruntime::NodeArg*> ArgMap;
|
||||
|
||||
std::shared_ptr<onnxruntime::Model> DummyGraphWithClip() {
|
||||
auto model = std::make_shared<onnxruntime::Model>("test", false, DefaultLoggingManager().DefaultLogger());
|
||||
onnxruntime::Graph& graph = model->MainGraph();
|
||||
TypeProto tensor_float;
|
||||
tensor_float.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT);
|
||||
onnxruntime::NodeArg input_def("X", &tensor_float), output_def("Y", &tensor_float);
|
||||
|
||||
graph.AddNode("node1", "Clip", "clip operator", ArgMap{&input_def}, ArgMap{&output_def});
|
||||
return model;
|
||||
}
|
||||
|
||||
std::unique_ptr<IExecutionProvider> CreateCPUExecutionProvider() {
|
||||
CPUExecutionProviderInfo info;
|
||||
return onnxruntime::make_unique<CPUExecutionProvider>(info);
|
||||
|
|
@ -69,8 +60,9 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) {
|
|||
std::unique_ptr<SequentialExecutionPlan> p_seq_exec_plan;
|
||||
// TODO below line is for testing only. In production use SequentialPlanner::CreatePlan()
|
||||
SequentialPlannerContext context(ExecutionMode::ORT_SEQUENTIAL);
|
||||
ASSERT_STATUS_OK(SequentialPlanner::CreatePlan(nullptr, GraphViewer(graph), {}, execution_providers, kernel_registry_manager,
|
||||
state.GetOrtValueNameIdxMap(), context, p_seq_exec_plan));
|
||||
ASSERT_STATUS_OK(SequentialPlanner::CreatePlan(nullptr, GraphViewer(graph), {}, execution_providers,
|
||||
kernel_registry_manager, state.GetOrtValueNameIdxMap(), context,
|
||||
p_seq_exec_plan));
|
||||
state.SetExecutionPlan(std::move(p_seq_exec_plan));
|
||||
|
||||
vector<OrtValue> outputs;
|
||||
|
|
@ -81,8 +73,9 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) {
|
|||
|
||||
TensorShape shape(std::vector<int64_t>{2, 3});
|
||||
OrtValue& mlvalue0 = *frame.GetMutableNodeInputOrOutputMLValue(start_index);
|
||||
const auto& memory_info = execution_providers.Get(xp_typ)->GetAllocator(0, OrtMemTypeDefault)->Info();
|
||||
ASSERT_STATUS_OK(frame.AllocateMLValueTensorSelfOwnBuffer(mlvalue0, start_index, DataTypeImpl::GetType<float>(),
|
||||
execution_providers.Get(xp_typ)->GetAllocator(0, OrtMemTypeDefault)->Info(), shape));
|
||||
memory_info, shape));
|
||||
|
||||
OrtValue* p_ml_value = frame.GetMutableNodeInputOrOutputMLValue(0);
|
||||
ASSERT_TRUE(p_ml_value != nullptr);
|
||||
|
|
@ -97,10 +90,10 @@ TEST_F(ExecutionFrameTest, TensorAllocationTest) {
|
|||
TensorShape shape2(std::vector<int64_t>{3, 2});
|
||||
OrtValue& mlvalue1 = *frame.GetMutableNodeInputOrOutputMLValue(start_index + 1);
|
||||
ASSERT_STATUS_OK(frame.AllocateMLValueTensorPreAllocateBuffer(mlvalue1,
|
||||
start_index,
|
||||
DataTypeImpl::GetType<float>(),
|
||||
p_tensor->Location(),
|
||||
shape2));
|
||||
start_index,
|
||||
DataTypeImpl::GetType<float>(),
|
||||
p_tensor->Location(),
|
||||
shape2));
|
||||
|
||||
const OrtValue* p_ml_value_const = frame.GetNodeInputOrOutputMLValue(1);
|
||||
auto tensor2 = p_ml_value_const ? &(p_ml_value_const->Get<Tensor>()) : nullptr;
|
||||
|
|
@ -224,8 +217,9 @@ TEST_F(ExecutionFrameTest, MemPatternTest) {
|
|||
|
||||
std::unique_ptr<SequentialExecutionPlan> p_seq_exec_plan = onnxruntime::make_unique<SequentialExecutionPlan>();
|
||||
SequentialPlannerContext context(ExecutionMode::ORT_SEQUENTIAL);
|
||||
ASSERT_STATUS_OK(SequentialPlanner::CreatePlan(nullptr, GraphViewer(graph), {}, execution_providers, kernel_registry_manager,
|
||||
mlvalue_name_idx_map, context, p_seq_exec_plan));
|
||||
ASSERT_STATUS_OK(SequentialPlanner::CreatePlan(nullptr, GraphViewer(graph), {}, execution_providers,
|
||||
kernel_registry_manager, mlvalue_name_idx_map, context,
|
||||
p_seq_exec_plan));
|
||||
|
||||
state.SetExecutionPlan(std::move(p_seq_exec_plan));
|
||||
|
||||
|
|
@ -237,19 +231,19 @@ TEST_F(ExecutionFrameTest, MemPatternTest) {
|
|||
OrtValue& mlvalue5 = *frame.GetMutableNodeInputOrOutputMLValue(5);
|
||||
|
||||
ASSERT_STATUS_OK(frame.AllocateMLValueTensorSelfOwnBuffer(mlvalue3, 3,
|
||||
DataTypeImpl::GetType<float>(),
|
||||
cpu_allocator->Info(),
|
||||
TensorShape(std::vector<int64_t>{2, 2})));
|
||||
DataTypeImpl::GetType<float>(),
|
||||
cpu_allocator->Info(),
|
||||
TensorShape(std::vector<int64_t>{2, 2})));
|
||||
|
||||
ASSERT_STATUS_OK(frame.AllocateMLValueTensorSelfOwnBuffer(mlvalue4, 4,
|
||||
DataTypeImpl::GetType<float>(),
|
||||
cpu_allocator->Info(),
|
||||
TensorShape(std::vector<int64_t>{2, 3})));
|
||||
DataTypeImpl::GetType<float>(),
|
||||
cpu_allocator->Info(),
|
||||
TensorShape(std::vector<int64_t>{2, 3})));
|
||||
|
||||
ASSERT_STATUS_OK(frame.AllocateMLValueTensorSelfOwnBuffer(mlvalue5, 5,
|
||||
DataTypeImpl::GetType<float>(),
|
||||
cpu_allocator->Info(),
|
||||
TensorShape(std::vector<int64_t>{2, 3})));
|
||||
DataTypeImpl::GetType<float>(),
|
||||
cpu_allocator->Info(),
|
||||
TensorShape(std::vector<int64_t>{2, 3})));
|
||||
MemoryPatternGroup pattern;
|
||||
ASSERT_STATUS_OK(frame.GeneratePatterns(&pattern));
|
||||
|
||||
|
|
@ -297,5 +291,64 @@ TEST(ExecutionFrameTestWithoutSessionState, BadModelInvalidDimParamUsage) {
|
|||
EXPECT_THAT(st.ErrorMessage(), testing::HasSubstr("Shape mismatch attempting to re-use buffer."));
|
||||
}
|
||||
|
||||
// Test that when an initializer is a graph output it is handled correctly
|
||||
TEST(ExecutionFrameTestInit, InitializerAsOutput) {
|
||||
const std::vector<float> expected{
|
||||
1.764052391052246f, 0.40015721321105957f, 0.978738009929657f, 2.2408931255340576f, 1.8675580024719238f,
|
||||
-0.9772778749465942f, 0.9500884413719177f, -0.15135720372200012f, -0.10321885347366333f, 0.4105985164642334f,
|
||||
0.14404356479644775f, 1.4542734622955322f, 0.7610377073287964f, 0.12167501449584961f, 0.44386324286460876f,
|
||||
0.3336743414402008f, 1.4940791130065918f, -0.2051582634449005f, 0.3130677044391632f, -0.8540957570075989f,
|
||||
-2.5529897212982178f, 0.653618574142456f, 0.8644362092018127f, -0.7421650290489197f, 2.269754648208618f};
|
||||
|
||||
SessionOptions so;
|
||||
|
||||
// test if pre-allocated fetch is provided the initializer values are copied into that buffer
|
||||
{
|
||||
InferenceSession session(so, GetEnvironment());
|
||||
ASSERT_STATUS_OK(session.Load(ORT_TSTR("testdata/initializer_as_output.onnx")));
|
||||
ASSERT_STATUS_OK(session.Initialize());
|
||||
|
||||
auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU);
|
||||
auto p_tensor = onnxruntime::make_unique<Tensor>(DataTypeImpl::GetType<float>(), TensorShape({5, 5}), allocator);
|
||||
const void* orig_buffer = p_tensor->DataRaw();
|
||||
|
||||
std::vector<OrtValue> results;
|
||||
results.resize(1);
|
||||
results[0].Init(p_tensor.release(), DataTypeImpl::GetType<Tensor>(),
|
||||
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
|
||||
RunOptions ro;
|
||||
ASSERT_STATUS_OK(session.Run(ro, {}, {}, {"values"}, &results));
|
||||
|
||||
EXPECT_EQ(results[0].Get<Tensor>().DataRaw(), orig_buffer);
|
||||
EXPECT_THAT(results[0].Get<Tensor>().DataAsSpan<float>(), ::testing::ContainerEq(gsl::make_span(expected)));
|
||||
}
|
||||
|
||||
// test that if no pre-allocated fetch is provided a new OrtValue is allocated for the results
|
||||
{
|
||||
class TestInferenceSesssion : public InferenceSession {
|
||||
public:
|
||||
TestInferenceSesssion(const SessionOptions& session_options,
|
||||
const Environment& session_env)
|
||||
: InferenceSession(session_options, session_env) {
|
||||
}
|
||||
|
||||
const SessionState& GetSessionState() const { return *session_state_; }
|
||||
};
|
||||
|
||||
TestInferenceSesssion session(so, GetEnvironment());
|
||||
ASSERT_STATUS_OK(session.Load(ORT_TSTR("testdata/initializer_as_output.onnx")));
|
||||
ASSERT_STATUS_OK(session.Initialize());
|
||||
|
||||
std::vector<OrtValue> results;
|
||||
RunOptions ro;
|
||||
ASSERT_STATUS_OK(session.Run(ro, {}, {}, {"values"}, &results));
|
||||
|
||||
// output buffer should not be the same as the initializer in SessionState
|
||||
const auto& initializers = session.GetSessionState().GetInitializedTensors();
|
||||
EXPECT_NE(results[0].Get<Tensor>().DataRaw(), initializers.at(0).Get<Tensor>().DataRaw());
|
||||
EXPECT_THAT(results[0].Get<Tensor>().DataAsSpan<float>(), ::testing::ContainerEq(gsl::make_span(expected)));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
7
onnxruntime/test/testdata/initializer_as_output.onnx
vendored
Normal file
7
onnxruntime/test/testdata/initializer_as_output.onnx
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
backend-test:Ç
|
||||
›values"Constant*†
|
||||
value*z"dxÌá?háÌ>“Žz?Ëj@$ ï?â.z¿ÿ8s?bý¾hdÓ½ø9Ò>(€>¢%º?^ÓB?À0ù= Bã>]ת>ü=¿?R¾iJ >¦Z¿/d#ÀŒS'?±K]?‡þ=¿©C@B const_tensor
test_constantb
|
||||
values
|
||||
|
||||
|
||||
B
|
||||
Loading…
Reference in a new issue