mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Avoid some heap allocations in the InferenceSession and Model classes (#3103)
* Avoid some heap allocations in the InferenceSession and Model classes
This commit is contained in:
Hariharan Seshadri
GitHub
parent
a02638eb46
commit
b8575dda7b
11 changed files with 210 additions and 185 deletions
|
|
@ -35,12 +35,11 @@ Model::Model(const std::string& graph_name,
|
|||
const std::vector<ONNX_NAMESPACE::FunctionProto>& model_functions,
|
||||
const logging::Logger& logger)
|
||||
: model_path_(Path::Parse(model_path)) {
|
||||
model_proto_ = onnxruntime::make_unique<ModelProto>();
|
||||
model_proto_->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
|
||||
model_proto_->mutable_graph()->set_name(graph_name);
|
||||
model_proto_.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
|
||||
model_proto_.mutable_graph()->set_name(graph_name);
|
||||
model_metadata_ = model_metadata;
|
||||
for (auto& metadata : model_metadata_) {
|
||||
const gsl::not_null<StringStringEntryProto*> prop{model_proto_->add_metadata_props()};
|
||||
const gsl::not_null<StringStringEntryProto*> prop{model_proto_.add_metadata_props()};
|
||||
prop->set_key(metadata.first);
|
||||
prop->set_value(metadata.second);
|
||||
}
|
||||
|
|
@ -58,52 +57,48 @@ Model::Model(const std::string& graph_name,
|
|||
}
|
||||
|
||||
for (const auto& domain : *p_domain_to_version) {
|
||||
const gsl::not_null<OperatorSetIdProto*> opset_id_proto{model_proto_->add_opset_import()};
|
||||
const gsl::not_null<OperatorSetIdProto*> opset_id_proto{model_proto_.add_opset_import()};
|
||||
opset_id_proto->set_domain(domain.first);
|
||||
opset_id_proto->set_version(domain.second);
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, const ONNX_NAMESPACE::FunctionProto*> model_functions_map;
|
||||
for (auto& func : model_functions) {
|
||||
auto func_ptr = model_proto_->add_functions();
|
||||
auto func_ptr = model_proto_.add_functions();
|
||||
func_ptr->CopyFrom(func);
|
||||
model_functions_map[func_ptr->name()] = func_ptr;
|
||||
}
|
||||
|
||||
// need to call private ctor so can't use make_shared
|
||||
GSL_SUPPRESS(r .11)
|
||||
graph_.reset(new Graph(*this, model_proto_->mutable_graph(), *p_domain_to_version, IrVersion(), schema_registry,
|
||||
graph_.reset(new Graph(*this, model_proto_.mutable_graph(), *p_domain_to_version, IrVersion(), schema_registry,
|
||||
logger, model_functions_map));
|
||||
}
|
||||
|
||||
Model::Model(const ModelProto& model_proto, const PathString& model_path,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger)
|
||||
: Model(onnxruntime::make_unique<ModelProto>(model_proto), model_path, local_registries, logger) {
|
||||
: Model(ModelProto(model_proto), model_path, local_registries, logger) {
|
||||
}
|
||||
|
||||
Model::Model(std::unique_ptr<ModelProto> model_proto, const PathString& model_path, const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
Model::Model(ModelProto&& model_proto, const PathString& model_path, const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger)
|
||||
: model_path_(Path::Parse(model_path)) {
|
||||
if (!model_proto) {
|
||||
throw std::invalid_argument("ModelProto was null.");
|
||||
}
|
||||
|
||||
if (!utils::HasGraph(*model_proto)) {
|
||||
if (!utils::HasGraph(model_proto)) {
|
||||
throw std::invalid_argument("ModelProto does not have a graph.");
|
||||
}
|
||||
|
||||
if (model_proto->opset_import_size() == 0) {
|
||||
if (model_proto.opset_import_size() == 0) {
|
||||
throw std::invalid_argument(
|
||||
"Missing opset in the model. All ModelProtos MUST have at least one entry that"
|
||||
" specifies which version of the ONNX OperatorSet is being imported.");
|
||||
}
|
||||
|
||||
if (!model_proto->has_ir_version() || model_proto->ir_version() > ONNX_NAMESPACE::Version::IR_VERSION) {
|
||||
if (!model_proto.has_ir_version() || model_proto.ir_version() > ONNX_NAMESPACE::Version::IR_VERSION) {
|
||||
throw std::invalid_argument("Unknown model file format version.");
|
||||
}
|
||||
|
||||
model_proto_ = std::move(model_proto);
|
||||
for (auto& prop : model_proto_->metadata_props()) {
|
||||
for (auto& prop : model_proto_.metadata_props()) {
|
||||
model_metadata_[prop.key()] = prop.value();
|
||||
}
|
||||
|
||||
|
|
@ -115,7 +110,7 @@ Model::Model(std::unique_ptr<ModelProto> model_proto, const PathString& model_pa
|
|||
}
|
||||
|
||||
std::unordered_map<std::string, int> domain_to_version;
|
||||
for (auto& opSet : model_proto_->opset_import()) {
|
||||
for (auto& opSet : model_proto_.opset_import()) {
|
||||
const auto& domain = opSet.domain();
|
||||
const auto version = opSet.version();
|
||||
// empty domain and 'ai.onnx' are equivalent
|
||||
|
|
@ -144,71 +139,71 @@ Model::Model(std::unique_ptr<ModelProto> model_proto, const PathString& model_pa
|
|||
for (const auto& domain : domain_map) {
|
||||
if (domain_to_version.find(domain.first) == domain_to_version.end()) {
|
||||
domain_to_version[domain.first] = domain.second;
|
||||
const gsl::not_null<OperatorSetIdProto*> opset_id_proto{model_proto_->add_opset_import()};
|
||||
const gsl::not_null<OperatorSetIdProto*> opset_id_proto{model_proto_.add_opset_import()};
|
||||
opset_id_proto->set_domain(domain.first);
|
||||
opset_id_proto->set_version(domain.second);
|
||||
}
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, const ONNX_NAMESPACE::FunctionProto*> model_functions_map;
|
||||
for (auto& func : model_proto_->functions()) {
|
||||
for (auto& func : model_proto_.functions()) {
|
||||
model_functions_map[func.name()] = &func;
|
||||
}
|
||||
|
||||
// create instance. need to call private ctor so can't use make_unique
|
||||
GSL_SUPPRESS(r .11)
|
||||
graph_.reset(new Graph(*this, model_proto_->mutable_graph(), domain_to_version, IrVersion(), schema_registry, logger,
|
||||
graph_.reset(new Graph(*this, model_proto_.mutable_graph(), domain_to_version, IrVersion(), schema_registry, logger,
|
||||
model_functions_map));
|
||||
}
|
||||
|
||||
Version Model::IrVersion() const {
|
||||
if (utils::HasIrVersion(*model_proto_)) {
|
||||
return model_proto_->ir_version();
|
||||
if (utils::HasIrVersion(model_proto_)) {
|
||||
return model_proto_.ir_version();
|
||||
}
|
||||
return kNoVersion;
|
||||
}
|
||||
|
||||
const std::string& Model::ProducerName() const {
|
||||
return model_proto_->producer_name();
|
||||
return model_proto_.producer_name();
|
||||
}
|
||||
|
||||
void Model::SetProducerName(const std::string& producer_name) {
|
||||
model_proto_->set_producer_name(producer_name);
|
||||
model_proto_.set_producer_name(producer_name);
|
||||
}
|
||||
|
||||
const std::string& Model::ProducerVersion() const {
|
||||
return model_proto_->producer_version();
|
||||
return model_proto_.producer_version();
|
||||
}
|
||||
|
||||
void Model::SetProducerVersion(const std::string& producer_version) {
|
||||
model_proto_->set_producer_version(producer_version);
|
||||
model_proto_.set_producer_version(producer_version);
|
||||
}
|
||||
|
||||
const std::string& Model::Domain() const {
|
||||
return model_proto_->domain();
|
||||
return model_proto_.domain();
|
||||
}
|
||||
|
||||
void Model::SetDomain(const std::string& domain) {
|
||||
model_proto_->set_domain(domain);
|
||||
model_proto_.set_domain(domain);
|
||||
}
|
||||
|
||||
Version Model::ModelVersion() const {
|
||||
if (utils::HasModelVersion(*model_proto_)) {
|
||||
return model_proto_->model_version();
|
||||
if (utils::HasModelVersion(model_proto_)) {
|
||||
return model_proto_.model_version();
|
||||
}
|
||||
return kNoVersion;
|
||||
}
|
||||
|
||||
void Model::SetModelversion(onnxruntime::Version version) {
|
||||
model_proto_->set_model_version(version);
|
||||
model_proto_.set_model_version(version);
|
||||
}
|
||||
|
||||
const std::string& Model::DocString() const {
|
||||
return model_proto_->doc_string();
|
||||
return model_proto_.doc_string();
|
||||
}
|
||||
|
||||
void Model::SetDocString(const std::string& doc_string) {
|
||||
model_proto_->set_doc_string(doc_string);
|
||||
model_proto_.set_doc_string(doc_string);
|
||||
}
|
||||
|
||||
const ModelMetaData& Model::MetaData() const noexcept {
|
||||
|
|
@ -224,14 +219,14 @@ const Graph& Model::MainGraph() const noexcept {
|
|||
}
|
||||
|
||||
void Model::AddFunction(const ONNX_NAMESPACE::FunctionProto& func_proto) {
|
||||
auto func_ptr = model_proto_->add_functions();
|
||||
auto func_ptr = model_proto_.add_functions();
|
||||
func_ptr->CopyFrom(func_proto);
|
||||
graph_->AddFunction(func_ptr);
|
||||
}
|
||||
|
||||
ModelProto Model::ToProto() {
|
||||
*(model_proto_->mutable_graph()) = graph_->ToGraphProto();
|
||||
return *model_proto_;
|
||||
*(model_proto_.mutable_graph()) = graph_->ToGraphProto();
|
||||
return model_proto_;
|
||||
}
|
||||
|
||||
Status Model::Load(std::istream& model_istream, ModelProto* p_model_proto) {
|
||||
|
|
@ -279,27 +274,27 @@ Status Model::Load(const ModelProto& model_proto,
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Model::Load(std::unique_ptr<ModelProto> p_model_proto,
|
||||
Status Model::Load(ModelProto&& model_proto,
|
||||
std::shared_ptr<Model>& model,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger) {
|
||||
return Model::Load(std::move(p_model_proto), PathString{}, model, local_registries, logger);
|
||||
return Model::Load(std::move(model_proto), PathString{}, model, local_registries, logger);
|
||||
}
|
||||
|
||||
Status Model::Load(std::unique_ptr<ModelProto> p_model_proto,
|
||||
Status Model::Load(ModelProto&& model_proto,
|
||||
const PathString& model_path,
|
||||
std::shared_ptr<Model>& model,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger) {
|
||||
// we expect a graph to be present
|
||||
if (!utils::HasGraph(*p_model_proto)) {
|
||||
if (!utils::HasGraph(model_proto)) {
|
||||
return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No graph was found in the protobuf.");
|
||||
}
|
||||
|
||||
// need to call private ctor so can't use make_shared
|
||||
GSL_SUPPRESS(r .11)
|
||||
try {
|
||||
model.reset(new Model(std::move(p_model_proto), model_path, local_registries, logger));
|
||||
model.reset(new Model(std::move(model_proto), model_path, local_registries, logger));
|
||||
} catch (const std::exception& ex) {
|
||||
return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Failed to load model with error: " + std::string(ex.what()));
|
||||
}
|
||||
|
|
@ -421,9 +416,9 @@ Status Model::LoadFromBytes(int count, void* p_bytes, /*out*/ std::shared_ptr<Mo
|
|||
Status Model::LoadFromBytes(int count, void* p_bytes, const PathString& model_path,
|
||||
std::shared_ptr<Model>& p_model, const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger) {
|
||||
auto model_proto = onnxruntime::make_unique<ModelProto>();
|
||||
ModelProto model_proto;
|
||||
|
||||
auto status = LoadFromBytes(count, p_bytes, *model_proto);
|
||||
auto status = LoadFromBytes(count, p_bytes, model_proto);
|
||||
if (!status.IsOK()) {
|
||||
return status;
|
||||
}
|
||||
|
|
@ -473,9 +468,9 @@ Status Model::Load(int fd, std::shared_ptr<Model>& p_model, const IOnnxRuntimeOp
|
|||
|
||||
Status Model::Load(int fd, const PathString& model_path, std::shared_ptr<Model>& p_model,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries, const logging::Logger& logger) {
|
||||
auto model_proto = onnxruntime::make_unique<ModelProto>();
|
||||
ModelProto model_proto;
|
||||
|
||||
ORT_RETURN_IF_ERROR(Load(fd, *model_proto));
|
||||
ORT_RETURN_IF_ERROR(Load(fd, model_proto));
|
||||
|
||||
p_model = std::make_shared<Model>(std::move(model_proto), model_path, local_registries, logger);
|
||||
|
||||
|
|
|
|||
|
|
@ -56,14 +56,14 @@ class Model {
|
|||
|
||||
// NOTE: after calling this constructor, <*this> model will
|
||||
// own the <model_proto>.
|
||||
explicit Model(std::unique_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
|
||||
explicit Model(ONNX_NAMESPACE::ModelProto&& model_proto,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger)
|
||||
: Model(std::move(model_proto), PathString(), local_registries, logger) {}
|
||||
|
||||
// NOTE: after calling this constructor, <*this> model will
|
||||
// own the <model_proto>.
|
||||
explicit Model(std::unique_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
|
||||
explicit Model(ONNX_NAMESPACE::ModelProto&& model_proto,
|
||||
const PathString& model_path,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger);
|
||||
|
|
@ -173,12 +173,12 @@ class Model {
|
|||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger);
|
||||
|
||||
static common::Status Load(std::unique_ptr<ONNX_NAMESPACE::ModelProto> p_model_proto,
|
||||
static common::Status Load(ONNX_NAMESPACE::ModelProto&& model_proto,
|
||||
/*out*/ std::shared_ptr<Model>& p_model,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
const logging::Logger& logger);
|
||||
|
||||
static common::Status Load(std::unique_ptr<ONNX_NAMESPACE::ModelProto> p_model_proto,
|
||||
static common::Status Load(ONNX_NAMESPACE::ModelProto&& model_proto,
|
||||
const PathString& model_path,
|
||||
/*out*/ std::shared_ptr<Model>& p_model,
|
||||
const IOnnxRuntimeOpSchemaRegistryList* local_registries,
|
||||
|
|
@ -186,7 +186,7 @@ class Model {
|
|||
|
||||
private:
|
||||
// Model data.
|
||||
std::unique_ptr<ONNX_NAMESPACE::ModelProto> model_proto_;
|
||||
ONNX_NAMESPACE::ModelProto model_proto_;
|
||||
|
||||
// This is a duplication of <model_proto_.metadata_props()>.
|
||||
// It gives better accessibility.
|
||||
|
|
|
|||
|
|
@ -3,11 +3,22 @@
|
|||
|
||||
#include "core/optimizer/graph_transformer_mgr.h"
|
||||
#include "core/optimizer/rule_based_graph_transformer.h"
|
||||
|
||||
using namespace onnxruntime;
|
||||
using namespace ::onnxruntime::common;
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
common::Status GraphTransformerManager::SetSteps(unsigned steps) {
|
||||
steps_ = steps;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
common::Status GraphTransformerManager::GetSteps(unsigned& steps) const {
|
||||
steps = steps_;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
common::Status GraphTransformerManager::ApplyTransformers(Graph& graph, TransformerLevel level, const logging::Logger& logger) const {
|
||||
const auto& transformers = level_to_transformer_map_.find(level);
|
||||
if (transformers == level_to_transformer_map_.end()) {
|
||||
|
|
@ -29,14 +40,14 @@ common::Status GraphTransformerManager::ApplyTransformers(Graph& graph, Transfor
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
common::Status GraphTransformerManager::Register(std::unique_ptr<GraphTransformer> transformer, TransformerLevel level){
|
||||
common::Status GraphTransformerManager::Register(std::unique_ptr<GraphTransformer> transformer, TransformerLevel level) {
|
||||
const auto& name = transformer->Name();
|
||||
if (transformers_info_.find(name) != transformers_info_.end()) {
|
||||
return Status(ONNXRUNTIME, FAIL, "This transformer is already registered " + name);
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "This transformer is already registered " + name);
|
||||
}
|
||||
|
||||
transformers_info_[name] = transformer.get();
|
||||
level_to_transformer_map_[level].push_back(std::move(transformer));
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -17,16 +17,20 @@ class GraphTransformerManager {
|
|||
explicit GraphTransformerManager(unsigned steps) : steps_(steps) {
|
||||
}
|
||||
|
||||
// Update (set) the maximum number of graph transformation steps
|
||||
common::Status SetSteps(unsigned steps);
|
||||
|
||||
// Get the maximum number of graph transformation steps
|
||||
common::Status GetSteps(unsigned& steps) const;
|
||||
|
||||
// Register a transformer with a level.
|
||||
common::Status Register(std::unique_ptr<GraphTransformer> transformer, TransformerLevel level);
|
||||
common::Status Register(std::unique_ptr<GraphTransformer> transformer, TransformerLevel level);
|
||||
|
||||
// Apply all transformers registered for the given level on the given graph
|
||||
common::Status ApplyTransformers(Graph& graph, TransformerLevel level, const logging::Logger& logger) const;
|
||||
|
||||
private:
|
||||
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformerManager);
|
||||
|
||||
const unsigned steps_;
|
||||
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformerManager);
|
||||
|
||||
// Older GCC versions don't support std::hash with enum types
|
||||
// Therefore, std::hash<T> appears to be undefined when T is an enum Type. This is fixed in version 6.1
|
||||
|
|
@ -38,7 +42,10 @@ class GraphTransformerManager {
|
|||
}
|
||||
};
|
||||
|
||||
// maximum number of graph transformation steps
|
||||
unsigned steps_;
|
||||
|
||||
std::unordered_map<TransformerLevel, std::vector<std::unique_ptr<GraphTransformer>>, EnumHashKey> level_to_transformer_map_;
|
||||
std::unordered_map<std::string, GraphTransformer*> transformers_info_;
|
||||
};
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -97,7 +97,8 @@ inline std::basic_string<T> GetCurrentTimeString() {
|
|||
std::atomic<uint32_t> InferenceSession::global_session_id_{1};
|
||||
|
||||
static Status FinalizeSessionOptions(const SessionOptions& user_provided_session_options,
|
||||
const ONNX_NAMESPACE::ModelProto* model_proto,
|
||||
const ONNX_NAMESPACE::ModelProto& model_proto,
|
||||
bool is_model_proto_parsed,
|
||||
/*out*/ SessionOptions& finalized_session_options) {
|
||||
const logging::Logger& default_logger = logging::LoggingManager::DefaultLogger();
|
||||
|
||||
|
|
@ -134,12 +135,12 @@ static Status FinalizeSessionOptions(const SessionOptions& user_provided_session
|
|||
|
||||
// In theory we should not hit this condition unless this internal class' APIs are being called incorrectly.
|
||||
// This is a good sanity check to enforce that the model has been parsed prior to looking into it for ort config.
|
||||
ORT_ENFORCE(model_proto, "Model needs to be provided to check for ORT config within it");
|
||||
ORT_ENFORCE(is_model_proto_parsed, "ModelProto needs to be parsed to check for ORT config within it");
|
||||
|
||||
// Use default logger as the session_logger_ hasn't been initialized yet.
|
||||
InferenceSessionUtils inference_session_utils(default_logger);
|
||||
|
||||
auto status = inference_session_utils.ParseOrtConfigJsonInModelProto(*model_proto);
|
||||
auto status = inference_session_utils.ParseOrtConfigJsonInModelProto(model_proto);
|
||||
if (!status.IsOK()) {
|
||||
return status;
|
||||
}
|
||||
|
|
@ -161,12 +162,13 @@ static Status FinalizeSessionOptions(const SessionOptions& user_provided_session
|
|||
|
||||
void InferenceSession::ConstructorCommon(const SessionOptions& session_options,
|
||||
logging::LoggingManager* logging_manager) {
|
||||
auto status = FinalizeSessionOptions(session_options, model_proto_.get(), session_options_);
|
||||
auto status = FinalizeSessionOptions(session_options, model_proto_, model_loaded_, session_options_);
|
||||
ORT_ENFORCE(status.IsOK(), "Could not finalize session options while constructing the inference session. Error Message: ",
|
||||
status.ErrorMessage());
|
||||
|
||||
graph_transformation_mgr_ = onnxruntime::make_unique<GraphTransformerManager>(
|
||||
session_options_.max_num_graph_transformation_steps);
|
||||
// Update the number of steps for the graph transformer manager using the "finalized" session options
|
||||
graph_transformation_mgr_.SetSteps(session_options_.max_num_graph_transformation_steps);
|
||||
|
||||
logging_manager_ = logging_manager;
|
||||
|
||||
thread_pool_ = concurrency::CreateThreadPool("intra_op_thread_pool",
|
||||
|
|
@ -199,7 +201,8 @@ void InferenceSession::ConstructorCommon(const SessionOptions& session_options,
|
|||
|
||||
InferenceSession::InferenceSession(const SessionOptions& session_options,
|
||||
logging::LoggingManager* logging_manager)
|
||||
: insert_cast_transformer_("CastFloat16Transformer") {
|
||||
: graph_transformation_mgr_(session_options.max_num_graph_transformation_steps),
|
||||
insert_cast_transformer_("CastFloat16Transformer") {
|
||||
// Initialize assets of this session instance
|
||||
ConstructorCommon(session_options, logging_manager);
|
||||
}
|
||||
|
|
@ -207,13 +210,13 @@ InferenceSession::InferenceSession(const SessionOptions& session_options,
|
|||
InferenceSession::InferenceSession(const SessionOptions& session_options,
|
||||
const std::string& model_uri,
|
||||
logging::LoggingManager* logging_manager)
|
||||
: insert_cast_transformer_("CastFloat16Transformer") {
|
||||
: graph_transformation_mgr_(session_options.max_num_graph_transformation_steps),
|
||||
insert_cast_transformer_("CastFloat16Transformer") {
|
||||
model_location_ = ToWideString(model_uri);
|
||||
model_proto_ = onnxruntime::make_unique<ONNX_NAMESPACE::ModelProto>();
|
||||
auto status = Model::Load(model_location_, *model_proto_);
|
||||
auto status = Model::Load(model_location_, model_proto_);
|
||||
ORT_ENFORCE(status.IsOK(), "Given model could not be parsed while creating inference session. Error message: ",
|
||||
status.ErrorMessage());
|
||||
|
||||
model_loaded_ = true;
|
||||
// Finalize session options and initialize assets of this session instance
|
||||
ConstructorCommon(session_options, logging_manager);
|
||||
}
|
||||
|
|
@ -222,13 +225,13 @@ InferenceSession::InferenceSession(const SessionOptions& session_options,
|
|||
InferenceSession::InferenceSession(const SessionOptions& session_options,
|
||||
const std::wstring& model_uri,
|
||||
logging::LoggingManager* logging_manager)
|
||||
: insert_cast_transformer_("CastFloat16Transformer") {
|
||||
: graph_transformation_mgr_(session_options.max_num_graph_transformation_steps),
|
||||
insert_cast_transformer_("CastFloat16Transformer") {
|
||||
model_location_ = ToWideString(model_uri);
|
||||
model_proto_ = onnxruntime::make_unique<ONNX_NAMESPACE::ModelProto>();
|
||||
auto status = Model::Load(model_location_, *model_proto_);
|
||||
auto status = Model::Load(model_location_, model_proto_);
|
||||
ORT_ENFORCE(status.IsOK(), "Given model could not be parsed while creating inference session. Error message: ",
|
||||
status.ErrorMessage());
|
||||
|
||||
model_loaded_ = true;
|
||||
// Finalize session options and initialize assets of this session instance
|
||||
ConstructorCommon(session_options, logging_manager);
|
||||
}
|
||||
|
|
@ -237,12 +240,12 @@ InferenceSession::InferenceSession(const SessionOptions& session_options,
|
|||
InferenceSession::InferenceSession(const SessionOptions& session_options,
|
||||
std::istream& model_istream,
|
||||
logging::LoggingManager* logging_manager)
|
||||
: insert_cast_transformer_("CastFloat16Transformer") {
|
||||
: graph_transformation_mgr_(session_options.max_num_graph_transformation_steps),
|
||||
insert_cast_transformer_("CastFloat16Transformer") {
|
||||
google::protobuf::io::IstreamInputStream zero_copy_input(&model_istream);
|
||||
model_proto_ = onnxruntime::make_unique<ONNX_NAMESPACE::ModelProto>();
|
||||
const bool result = model_proto_->ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof();
|
||||
const bool result = model_proto_.ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof();
|
||||
ORT_ENFORCE(result, "Could not parse model successfully while constructing the inference session");
|
||||
|
||||
model_loaded_ = true;
|
||||
// Finalize session options and initialize assets of this session instance
|
||||
ConstructorCommon(session_options, logging_manager);
|
||||
}
|
||||
|
|
@ -251,11 +254,11 @@ InferenceSession::InferenceSession(const SessionOptions& session_options,
|
|||
const void* model_data,
|
||||
int model_data_len,
|
||||
logging::LoggingManager* logging_manager)
|
||||
: insert_cast_transformer_("CastFloat16Transformer") {
|
||||
model_proto_ = onnxruntime::make_unique<ONNX_NAMESPACE::ModelProto>();
|
||||
const bool result = model_proto_->ParseFromArray(model_data, model_data_len);
|
||||
: graph_transformation_mgr_(session_options.max_num_graph_transformation_steps),
|
||||
insert_cast_transformer_("CastFloat16Transformer") {
|
||||
const bool result = model_proto_.ParseFromArray(model_data, model_data_len);
|
||||
ORT_ENFORCE(result, "Could not parse model successfully while constructing the inference session");
|
||||
|
||||
model_loaded_ = true;
|
||||
// Finalize session options and initialize assets of this session instance
|
||||
ConstructorCommon(session_options, logging_manager);
|
||||
}
|
||||
|
|
@ -315,7 +318,7 @@ common::Status InferenceSession::RegisterGraphTransformer(
|
|||
if (p_graph_transformer == nullptr) {
|
||||
return Status(common::ONNXRUNTIME, common::FAIL, "Received nullptr for graph transformer");
|
||||
}
|
||||
return graph_transformation_mgr_->Register(std::move(p_graph_transformer), level);
|
||||
return graph_transformation_mgr_.Register(std::move(p_graph_transformer), level);
|
||||
}
|
||||
|
||||
common::Status InferenceSession::AddCustomTransformerList(const std::vector<std::string>& transformers_to_enable) {
|
||||
|
|
@ -372,11 +375,6 @@ common::Status InferenceSession::Load(std::function<common::Status(std::shared_p
|
|||
// all steps complete, mark the model as loaded.
|
||||
is_model_loaded_ = true;
|
||||
|
||||
// model_proto_ should either - 1) always have been a nullptr if the ModelProto was never parsed in the ctor (or)
|
||||
// 2) should have become a nullptr by passing on the ownership of the ModelProto resource it was pointing to,
|
||||
// to the Model instance
|
||||
ORT_ENFORCE(model_proto_ == nullptr, "Failed to clear up model_proto_ in Inference Session");
|
||||
|
||||
telemetry_.event_name_ = event_name;
|
||||
|
||||
} catch (const std::exception& ex) {
|
||||
|
|
@ -417,7 +415,7 @@ common::Status InferenceSession::Load(const std::basic_string<T>& model_uri) {
|
|||
}
|
||||
|
||||
common::Status InferenceSession::Load(const std::string& model_uri) {
|
||||
if (model_proto_ != nullptr) {
|
||||
if (model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has already been parsed. "
|
||||
"Invoke Load().");
|
||||
|
|
@ -428,7 +426,7 @@ common::Status InferenceSession::Load(const std::string& model_uri) {
|
|||
|
||||
#ifdef _WIN32
|
||||
common::Status InferenceSession::Load(const std::wstring& model_uri) {
|
||||
if (model_proto_ != nullptr) {
|
||||
if (model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has already been parsed. "
|
||||
"Invoke Load().");
|
||||
|
|
@ -439,7 +437,7 @@ common::Status InferenceSession::Load(const std::wstring& model_uri) {
|
|||
#endif
|
||||
|
||||
common::Status InferenceSession::Load(const ModelProto& model_proto) {
|
||||
if (model_proto_ != nullptr) {
|
||||
if (model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has already been parsed. "
|
||||
"Invoke Load().");
|
||||
|
|
@ -461,7 +459,7 @@ common::Status InferenceSession::Load(const ModelProto& model_proto) {
|
|||
}
|
||||
|
||||
common::Status InferenceSession::Load(std::unique_ptr<ModelProto> p_model_proto) {
|
||||
if (model_proto_ != nullptr) {
|
||||
if (model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has already been parsed. "
|
||||
"Invoke Load().");
|
||||
|
|
@ -474,7 +472,7 @@ common::Status InferenceSession::Load(std::unique_ptr<ModelProto> p_model_proto)
|
|||
AddCustomOpDomains({domain.get()});
|
||||
}
|
||||
#endif
|
||||
return onnxruntime::Model::Load(std::move(p_model_proto), PathString(), model,
|
||||
return onnxruntime::Model::Load(std::move(*p_model_proto), PathString(), model,
|
||||
HasLocalSchema() ? &custom_schema_registries_ : nullptr, *session_logger_);
|
||||
};
|
||||
|
||||
|
|
@ -482,23 +480,23 @@ common::Status InferenceSession::Load(std::unique_ptr<ModelProto> p_model_proto)
|
|||
}
|
||||
|
||||
common::Status InferenceSession::Load(std::istream& model_istream) {
|
||||
if (model_proto_ != nullptr) {
|
||||
if (model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has already been parsed. "
|
||||
"Invoke Load().");
|
||||
}
|
||||
|
||||
auto loader = [this, &model_istream](std::shared_ptr<onnxruntime::Model>& model) {
|
||||
auto model_proto = onnxruntime::make_unique<ONNX_NAMESPACE::ModelProto>();
|
||||
ModelProto model_proto;
|
||||
|
||||
google::protobuf::io::IstreamInputStream zero_copy_input(&model_istream);
|
||||
const bool result = model_proto->ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof();
|
||||
const bool result = model_proto.ParseFromZeroCopyStream(&zero_copy_input) && model_istream.eof();
|
||||
if (!result) {
|
||||
return Status(common::ONNXRUNTIME, common::INVALID_PROTOBUF,
|
||||
"Failed to load model because protobuf parsing failed.");
|
||||
}
|
||||
#ifdef ENABLE_LANGUAGE_INTEROP_OPS
|
||||
LoadInterOp(*model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; });
|
||||
LoadInterOp(model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; });
|
||||
for (const auto& domain : interop_domains_) {
|
||||
AddCustomOpDomains({domain.get()});
|
||||
}
|
||||
|
|
@ -511,22 +509,22 @@ common::Status InferenceSession::Load(std::istream& model_istream) {
|
|||
}
|
||||
|
||||
common::Status InferenceSession::Load(const void* model_data, int model_data_len) {
|
||||
if (model_proto_ != nullptr) {
|
||||
if (model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has already been parsed. "
|
||||
"Invoke Load().");
|
||||
}
|
||||
|
||||
auto loader = [this, model_data, model_data_len](std::shared_ptr<onnxruntime::Model>& model) {
|
||||
auto model_proto = onnxruntime::make_unique<ONNX_NAMESPACE::ModelProto>();
|
||||
ModelProto model_proto;
|
||||
|
||||
const bool result = model_proto->ParseFromArray(model_data, model_data_len);
|
||||
const bool result = model_proto.ParseFromArray(model_data, model_data_len);
|
||||
if (!result) {
|
||||
return Status(common::ONNXRUNTIME, common::INVALID_PROTOBUF,
|
||||
"Failed to load model because protobuf parsing failed.");
|
||||
}
|
||||
#ifdef ENABLE_LANGUAGE_INTEROP_OPS
|
||||
LoadInterOp(*model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; });
|
||||
LoadInterOp(model_proto, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; });
|
||||
for (const auto& domain : interop_domains_) {
|
||||
AddCustomOpDomains({domain.get()});
|
||||
}
|
||||
|
|
@ -540,7 +538,7 @@ common::Status InferenceSession::Load(const void* model_data, int model_data_len
|
|||
}
|
||||
|
||||
common::Status InferenceSession::Load() {
|
||||
if (model_proto_ == nullptr) {
|
||||
if (!model_loaded_) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"ModelProto corresponding to the model to be loaded has not been parsed yet. "
|
||||
"This API should be called in conjunction with a ctor that takes a model abstraction.");
|
||||
|
|
@ -548,7 +546,7 @@ common::Status InferenceSession::Load() {
|
|||
|
||||
auto loader = [this](std::shared_ptr<onnxruntime::Model>& model) {
|
||||
#ifdef ENABLE_LANGUAGE_INTEROP_OPS
|
||||
LoadInterOp(*this->model_proto_, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; });
|
||||
LoadInterOp(this->model_proto_, interop_domains_, [&](const char* msg) { LOGS(*session_logger_, WARNING) << msg; });
|
||||
for (const auto& domain : interop_domains_) {
|
||||
AddCustomOpDomains({domain.get()});
|
||||
}
|
||||
|
|
@ -823,7 +821,7 @@ common::Status InferenceSession::Initialize() {
|
|||
}
|
||||
|
||||
// add predefined transformers
|
||||
AddPredefinedTransformers(*graph_transformation_mgr_, session_options_.graph_optimization_level,
|
||||
AddPredefinedTransformers(graph_transformation_mgr_, session_options_.graph_optimization_level,
|
||||
transformers_to_enable_);
|
||||
|
||||
onnxruntime::Graph& graph = model_->MainGraph();
|
||||
|
|
@ -845,7 +843,7 @@ common::Status InferenceSession::Initialize() {
|
|||
ORT_RETURN_IF_ERROR_SESSIONID_(CreateSubgraphSessionState(graph, *session_state_));
|
||||
|
||||
// apply any transformations to the main graph and any subgraphs
|
||||
ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, *graph_transformation_mgr_,
|
||||
ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, graph_transformation_mgr_,
|
||||
execution_providers_, kernel_registry_manager_,
|
||||
insert_cast_transformer_,
|
||||
*session_state_));
|
||||
|
|
|
|||
|
|
@ -441,7 +441,7 @@ class InferenceSession {
|
|||
|
||||
SessionOptions session_options_;
|
||||
|
||||
std::unique_ptr<onnxruntime::GraphTransformerManager> graph_transformation_mgr_;
|
||||
onnxruntime::GraphTransformerManager graph_transformation_mgr_;
|
||||
|
||||
// List of transformers to run. When this list is not empty only the transformers in this list
|
||||
// will be run regardless of the level set.
|
||||
|
|
@ -510,20 +510,20 @@ class InferenceSession {
|
|||
InterOpDomains interop_domains_;
|
||||
#endif
|
||||
// used to support platform telemetry
|
||||
static std::atomic<uint32_t> global_session_id_; // a monotonically increasing session id
|
||||
uint32_t session_id_; // the current session's id
|
||||
static std::atomic<uint32_t> global_session_id_; // a monotonically increasing session id
|
||||
uint32_t session_id_; // the current session's id
|
||||
|
||||
struct Telemetry {
|
||||
Telemetry() : time_sent_last_(), time_sent_last_evalutation_start_() {}
|
||||
uint32_t total_runs_since_last_ = 0; // the total number of Run() calls since the last report
|
||||
long long total_run_duration_since_last_ = 0; // the total duration (us) of Run() calls since the last report
|
||||
std::string event_name_; // where the model is loaded from: ["model_loading_uri", "model_loading_proto", "model_loading_istream"]
|
||||
uint32_t total_runs_since_last_ = 0; // the total number of Run() calls since the last report
|
||||
long long total_run_duration_since_last_ = 0; // the total duration (us) of Run() calls since the last report
|
||||
std::string event_name_; // where the model is loaded from: ["model_loading_uri", "model_loading_proto", "model_loading_istream"]
|
||||
|
||||
TimePoint time_sent_last_; // the TimePoint of the last report
|
||||
TimePoint time_sent_last_; // the TimePoint of the last report
|
||||
TimePoint time_sent_last_evalutation_start_;
|
||||
// Event Rate per provider < 20 peak events per second
|
||||
constexpr static long long kDurationBetweenSending = 1000 * 1000 * 60 * 10; // duration in (us). send a report every 10 mins
|
||||
constexpr static long long kDurationBetweenSendingEvaluationStart = 1000 * 50; // duration in (us). send a EvaluationStop Event every 50 ms;
|
||||
// Event Rate per provider < 20 peak events per second
|
||||
constexpr static long long kDurationBetweenSending = 1000 * 1000 * 60 * 10; // duration in (us). send a report every 10 mins
|
||||
constexpr static long long kDurationBetweenSendingEvaluationStart = 1000 * 50; // duration in (us). send a EvaluationStop Event every 50 ms;
|
||||
|
||||
bool isEvaluationStart = false;
|
||||
} telemetry_;
|
||||
|
|
@ -534,6 +534,8 @@ class InferenceSession {
|
|||
#endif
|
||||
|
||||
// used to hold the ModelProto parsed in an applicable ctor to be used while calling parameter-less Load()
|
||||
std::unique_ptr<ONNX_NAMESPACE::ModelProto> model_proto_;
|
||||
ONNX_NAMESPACE::ModelProto model_proto_;
|
||||
|
||||
bool model_loaded_ = false;
|
||||
};
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -162,76 +162,76 @@ static void ConstructASimpleAddGraph(GraphProto& g, const char* domain) {
|
|||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddWithoutDomain) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "", 10);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), nullptr);
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "", 10);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), nullptr);
|
||||
std::shared_ptr<Model> model;
|
||||
ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_));
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddDefaultDomain) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "", 10);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), "");
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "", 10);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), "");
|
||||
std::shared_ptr<Model> model;
|
||||
ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_));
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddFutureOpSet) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "", 9999);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), "ai.onnx");
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "", 9999);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), "ai.onnx");
|
||||
std::shared_ptr<Model> model;
|
||||
Status st;
|
||||
ASSERT_FALSE((st = Model::Load(std::move(m), model, nullptr, *logger_)).IsOK());
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddONNXDomain) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "", 10);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), "ai.onnx");
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "", 10);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), "ai.onnx");
|
||||
std::shared_ptr<Model> model;
|
||||
ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_));
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddONNXDomain2) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "ai.onnx", 10);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), "ai.onnx");
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "ai.onnx", 10);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), "ai.onnx");
|
||||
std::shared_ptr<Model> model;
|
||||
ASSERT_STATUS_OK(Model::Load(std::move(m), model, nullptr, *logger_));
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddWrongDomain) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "", 10);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), "AAAA");
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "", 10);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), "AAAA");
|
||||
std::shared_ptr<Model> model;
|
||||
Status st;
|
||||
ASSERT_FALSE((st = Model::Load(std::move(m), model, nullptr, *logger_)).IsOK());
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleAddWrongDomain2) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "AAAA", 10);
|
||||
ConstructASimpleAddGraph(*m->mutable_graph(), "AAAA");
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "AAAA", 10);
|
||||
ConstructASimpleAddGraph(*m.mutable_graph(), "AAAA");
|
||||
std::shared_ptr<Model> model;
|
||||
Status st;
|
||||
ASSERT_FALSE((st = Model::Load(std::move(m), model, nullptr, *logger_)).IsOK());
|
||||
}
|
||||
|
||||
TEST_F(GraphTest, SimpleUnique) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "", 11);
|
||||
GraphProto& g = *m->mutable_graph();
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "", 11);
|
||||
GraphProto& g = *m.mutable_graph();
|
||||
NodeProto* node = g.add_node();
|
||||
*node->add_input() = "x";
|
||||
*node->add_output() = "sum";
|
||||
|
|
@ -248,11 +248,11 @@ TEST_F(GraphTest, SimpleUnique) {
|
|||
}
|
||||
|
||||
TEST_F(GraphTest, WrongOpset) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
//No Op registered for Unique with domain_version of 1
|
||||
ImportOpset(*m, "", 1);
|
||||
GraphProto& g = *m->mutable_graph();
|
||||
ImportOpset(m, "", 1);
|
||||
GraphProto& g = *m.mutable_graph();
|
||||
NodeProto* node = g.add_node();
|
||||
*node->add_input() = "x";
|
||||
*node->add_output() = "sum";
|
||||
|
|
@ -270,11 +270,11 @@ TEST_F(GraphTest, WrongOpset) {
|
|||
}
|
||||
|
||||
TEST_F(GraphTest, ExtraInput) {
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
//Node () has input size 2 not in range [min=1, max=1].
|
||||
ImportOpset(*m, "", 11);
|
||||
GraphProto& g = *m->mutable_graph();
|
||||
ImportOpset(m, "", 11);
|
||||
GraphProto& g = *m.mutable_graph();
|
||||
NodeProto* node = g.add_node();
|
||||
*node->add_input() = "x";
|
||||
*node->add_input() = "y";
|
||||
|
|
@ -297,10 +297,10 @@ TEST_F(GraphTest, LocalCustomRegistry) {
|
|||
std::vector<ONNX_NAMESPACE::OpSchema> schema = {
|
||||
OpSchema().SetName("FakeUnique").Input(0, "X", "A N-D input tensor that is to be processed.", "T").Output(0, "Y", "desc", "T").TypeConstraint("T", OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.").SetDomain("FakeTestDomain")};
|
||||
ASSERT_TRUE(registry->RegisterOpSet(schema, "FakeTestDomain", 0, 1).IsOK());
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ImportOpset(*m, "FakeTestDomain", 1);
|
||||
GraphProto& g = *m->mutable_graph();
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
ImportOpset(m, "FakeTestDomain", 1);
|
||||
GraphProto& g = *m.mutable_graph();
|
||||
NodeProto* node = g.add_node();
|
||||
*node->add_input() = "x";
|
||||
*node->add_output() = "sum";
|
||||
|
|
@ -323,11 +323,11 @@ TEST_F(GraphTest, LocalCustomRegistryWrongOpsetImportVersion) {
|
|||
std::vector<ONNX_NAMESPACE::OpSchema> schema = {
|
||||
OpSchema().SetName("FakeUnique").Input(0, "X", "A N-D input tensor that is to be processed.", "T").Output(0, "Y", "desc", "T").TypeConstraint("T", OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.").SetDomain("FakeTestDomain")};
|
||||
ASSERT_TRUE(registry->RegisterOpSet(schema, "FakeTestDomain", 0, 1).IsOK());
|
||||
std::unique_ptr<ModelProto> m = onnxruntime::make_unique<ModelProto>();
|
||||
m->set_ir_version(3);
|
||||
ModelProto m;
|
||||
m.set_ir_version(3);
|
||||
//Should be 1, but we put 11 herer so the model loading will fail
|
||||
ImportOpset(*m, "FakeTestDomain", 11);
|
||||
GraphProto& g = *m->mutable_graph();
|
||||
ImportOpset(m, "FakeTestDomain", 11);
|
||||
GraphProto& g = *m.mutable_graph();
|
||||
NodeProto* node = g.add_node();
|
||||
*node->add_input() = "x";
|
||||
*node->add_output() = "sum";
|
||||
|
|
|
|||
|
|
@ -142,8 +142,8 @@ TEST_P(ONNXModelsTest1, LoadFromProtobuf) {
|
|||
std::unique_ptr<ZeroCopyInputStream> raw_input(new FileInputStream(fd));
|
||||
std::unique_ptr<CodedInputStream> coded_input(new CodedInputStream(raw_input.get()));
|
||||
coded_input->SetTotalBytesLimit(INT_MAX);
|
||||
std::unique_ptr<ModelProto> model_proto = onnxruntime::make_unique<ModelProto>();
|
||||
bool result = model_proto->ParseFromCodedStream(coded_input.get());
|
||||
ModelProto model_proto;
|
||||
bool result = model_proto.ParseFromCodedStream(coded_input.get());
|
||||
coded_input.reset();
|
||||
raw_input.reset();
|
||||
ASSERT_TRUE(result);
|
||||
|
|
|
|||
|
|
@ -30,10 +30,10 @@ TEST(FreeDimensionOverrideTransformerTest, Test) {
|
|||
// and DATA_CHANNEL. Supplying these overrides to the transformer should replace those free
|
||||
// dimensions with values of 1 and 42, respectively.
|
||||
std::vector<FreeDimensionOverride> overrides =
|
||||
{
|
||||
FreeDimensionOverride{ onnx::DATA_BATCH, 1 },
|
||||
FreeDimensionOverride{ onnx::DATA_CHANNEL, 42 },
|
||||
};
|
||||
{
|
||||
FreeDimensionOverride{onnx::DATA_BATCH, 1},
|
||||
FreeDimensionOverride{onnx::DATA_CHANNEL, 42},
|
||||
};
|
||||
|
||||
auto graph_transformer = onnxruntime::make_unique<FreeDimensionOverrideTransformer>(overrides);
|
||||
|
||||
|
|
@ -46,10 +46,10 @@ TEST(FreeDimensionOverrideTransformerTest, Test) {
|
|||
// Verify that the shape of the input graph has the correct values
|
||||
|
||||
const auto& graph_inputs = graph.GetInputs();
|
||||
ASSERT_TRUE(graph_inputs.size() == 1); // This model only has a single input ('x')
|
||||
ASSERT_TRUE(graph_inputs.size() == 1); // This model only has a single input ('x')
|
||||
|
||||
const auto* input_shape = graph_inputs[0]->Shape();
|
||||
ASSERT_TRUE(input_shape->dim_size() == 3); // Model takes a 3D tensor as input; two of those dimensions are (were) free dimensions
|
||||
ASSERT_TRUE(input_shape->dim_size() == 3); // Model takes a 3D tensor as input; two of those dimensions are (were) free dimensions
|
||||
|
||||
ASSERT_TRUE(input_shape->dim(0).denotation() == onnx::DATA_BATCH);
|
||||
ASSERT_TRUE(input_shape->dim(0).has_dim_value());
|
||||
|
|
|
|||
|
|
@ -1502,22 +1502,22 @@ static void TestSkipLayerNormFusion(const std::basic_string<ORTCHAR_T>& file_pat
|
|||
|
||||
std::map<std::string, int> op_to_count = CountOpsInGraph(graph);
|
||||
ASSERT_TRUE(op_to_count["Div"] == 0);
|
||||
ASSERT_TRUE(op_to_count["Add"] == add_count );
|
||||
ASSERT_TRUE(op_to_count["Add"] == add_count);
|
||||
ASSERT_TRUE(op_to_count["Sub"] == 0);
|
||||
ASSERT_TRUE(op_to_count["ReduceMean"] == 0);
|
||||
ASSERT_TRUE(op_to_count["Pow"] == 0);
|
||||
ASSERT_TRUE(op_to_count["Sqrt"] == 0);
|
||||
ASSERT_TRUE(op_to_count["LayerNormalization"] == ln_count );
|
||||
ASSERT_TRUE(op_to_count["SkipLayerNormalization"] == skip_ln_count );
|
||||
ASSERT_TRUE(op_to_count["LayerNormalization"] == ln_count);
|
||||
ASSERT_TRUE(op_to_count["SkipLayerNormalization"] == skip_ln_count);
|
||||
}
|
||||
|
||||
TEST(GraphTransformationTests, SkipLayerNormFusionTest) {
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format1.onnx", 0, 0, 1);
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format2.onnx", 0, 0, 1 );
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format3.onnx", 0, 0, 1 );
|
||||
TestSkipLayerNormFusion( MODEL_FOLDER "fusion/skip_layer_norm_format1_partial.onnx", 1, 0, 1 );
|
||||
TestSkipLayerNormFusion( MODEL_FOLDER "fusion/skip_layer_norm_format2_partial.onnx", 1, 0, 1 );
|
||||
TestSkipLayerNormFusion( MODEL_FOLDER "fusion/skip_layer_norm_format3_no_fusion.onnx", 1, 1, 0 );
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format2.onnx", 0, 0, 1);
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format3.onnx", 0, 0, 1);
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format1_partial.onnx", 1, 0, 1);
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format2_partial.onnx", 1, 0, 1);
|
||||
TestSkipLayerNormFusion(MODEL_FOLDER "fusion/skip_layer_norm_format3_no_fusion.onnx", 1, 1, 0);
|
||||
}
|
||||
|
||||
TEST(GraphTransformationTests, EmbedLayerNormFusionFormat1) {
|
||||
|
|
@ -1666,4 +1666,4 @@ TEST(GraphTransformationTests, EmbedLayerNormFusionFormat5) {
|
|||
#endif
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -25,14 +25,14 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) {
|
|||
DefaultLoggingManager().DefaultLogger())
|
||||
.IsOK());
|
||||
Graph& graph = model->MainGraph();
|
||||
|
||||
|
||||
// Create rule based transformer with a dummy rewrite rule and register it with Cuda as compatible provider
|
||||
std::unordered_set<std::string> compatible_provider{onnxruntime::kCudaExecutionProvider};
|
||||
auto dummy_rule = onnxruntime::make_unique<DummyRewriteRule>("DummyRule");
|
||||
const auto* dummy_rule_ptr = dummy_rule.get();
|
||||
|
||||
auto graph_transformer = onnxruntime::make_unique<RuleBasedGraphTransformer>("CUDATopDownTransformer", compatible_provider);
|
||||
graph_transformer->Register(std::move(dummy_rule));
|
||||
graph_transformer->Register(std::move(dummy_rule));
|
||||
|
||||
// Create rule based transformer with a dummy rewrite rule and register it with CPU as compatible provider
|
||||
auto dummy_rule1 = onnxruntime::make_unique<DummyRewriteRule>("DummyRule1");
|
||||
|
|
@ -40,7 +40,7 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) {
|
|||
|
||||
auto graph_transformer1 = onnxruntime::make_unique<RuleBasedGraphTransformer>("CPUTopDownTransformer");
|
||||
|
||||
graph_transformer1->Register(std::move(dummy_rule1));
|
||||
graph_transformer1->Register(std::move(dummy_rule1));
|
||||
|
||||
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
|
||||
graph_transformation_mgr.Register(std::move(graph_transformer), TransformerLevel::Level2);
|
||||
|
|
@ -56,5 +56,17 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) {
|
|||
ASSERT_TRUE(dummy_rule1_ptr->IsRewriteRuleInvoked());
|
||||
}
|
||||
|
||||
TEST(RuleBasedGraphTransformerTest, TestSettingStepsInGraphTransformerManager) {
|
||||
// steps provided at object construction time
|
||||
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
|
||||
unsigned steps_queried;
|
||||
graph_transformation_mgr.GetSteps(steps_queried);
|
||||
ASSERT_EQ(steps_queried, static_cast<unsigned>(5));
|
||||
|
||||
// steps upadted
|
||||
graph_transformation_mgr.SetSteps(10);
|
||||
graph_transformation_mgr.GetSteps(steps_queried);
|
||||
ASSERT_EQ(steps_queried, static_cast<unsigned> (10));
|
||||
}
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
} // namespace onnxruntime
|
||||
Loading…
Reference in a new issue