diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index d61396694e..7369925e9d 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -486,7 +486,7 @@ common::Status InferenceSession::SaveToOrtFormat(const std::basic_string load_ort_format_mo } ORT_RETURN_IF_ERROR(load_ort_format_model_bytes()); + + // Verify the ort_format_model_bytes_ is a valid InferenceSessionBuffer before we access the data + flatbuffers::Verifier verifier(ort_format_model_bytes_.data(), ort_format_model_bytes_.size()); + ORT_RETURN_IF_NOT(fbs::VerifyInferenceSessionBuffer(verifier)); + const auto* fbs_session = fbs::GetInferenceSession(ort_format_model_bytes_.data()); ORT_RETURN_IF(nullptr == fbs_session, "InferenceSession is null. Invalid ORT format model."); @@ -1154,676 +1159,676 @@ common::Status InferenceSession::Initialize() { } #endif // !defined(ORT_MINIMAL_BUILD) - session_state_->ResolveMemoryPatternFlag(); - is_inited_ = true; + session_state_->ResolveMemoryPatternFlag(); + is_inited_ = true; - // we don't directly use the ORT format bytes currently, so free those now - std::vector().swap(ort_format_model_bytes_); + // we don't directly use the ORT format bytes currently, so free those now + std::vector().swap(ort_format_model_bytes_); - // and log telemetry - bool model_has_fp16_inputs = ModelHasFP16Inputs(graph); - env.GetTelemetryProvider().LogSessionCreation( - session_id_, model_->IrVersion(), model_->ProducerName(), model_->ProducerVersion(), model_->Domain(), - model_->MainGraph().DomainToVersionMap(), model_->MainGraph().Name(), model_->MetaData(), - telemetry_.event_name_, execution_providers_.GetIds(), model_has_fp16_inputs); - LOGS(*session_logger_, INFO) << "Session successfully initialized."; - } - ORT_CATCH(const NotImplementedException& ex) { - ORT_HANDLE_EXCEPTION([&]() { - status = ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Exception during initialization: ", ex.what()); - LOGS(*session_logger_, ERROR) << status.ErrorMessage(); - }); - } - ORT_CATCH(const std::exception& ex) { - ORT_HANDLE_EXCEPTION([&]() { - status = ORT_MAKE_STATUS(ONNXRUNTIME, RUNTIME_EXCEPTION, "Exception during initialization: ", ex.what()); - LOGS(*session_logger_, ERROR) << status.ErrorMessage(); - }); - } - ORT_CATCH(...) { - status = ORT_MAKE_STATUS(ONNXRUNTIME, RUNTIME_EXCEPTION, "Encountered unknown exception in Initialize()"); + // and log telemetry + bool model_has_fp16_inputs = ModelHasFP16Inputs(graph); + env.GetTelemetryProvider().LogSessionCreation( + session_id_, model_->IrVersion(), model_->ProducerName(), model_->ProducerVersion(), model_->Domain(), + model_->MainGraph().DomainToVersionMap(), model_->MainGraph().Name(), model_->MetaData(), + telemetry_.event_name_, execution_providers_.GetIds(), model_has_fp16_inputs); + LOGS(*session_logger_, INFO) << "Session successfully initialized."; + } + ORT_CATCH(const NotImplementedException& ex) { + ORT_HANDLE_EXCEPTION([&]() { + status = ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Exception during initialization: ", ex.what()); LOGS(*session_logger_, ERROR) << status.ErrorMessage(); - } - - if (session_profiler_.IsEnabled()) { - session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "session_initialization", tp); - } - - if (status.IsOK()) { - for (auto& xp : execution_providers_) { - auto end_status = xp->OnSessionInitializationEnd(); - if (status.IsOK()) { - status = end_status; - } - } - } - - return status; + }); + } + ORT_CATCH(const std::exception& ex) { + ORT_HANDLE_EXCEPTION([&]() { + status = ORT_MAKE_STATUS(ONNXRUNTIME, RUNTIME_EXCEPTION, "Exception during initialization: ", ex.what()); + LOGS(*session_logger_, ERROR) << status.ErrorMessage(); + }); + } + ORT_CATCH(...) { + status = ORT_MAKE_STATUS(ONNXRUNTIME, RUNTIME_EXCEPTION, "Encountered unknown exception in Initialize()"); + LOGS(*session_logger_, ERROR) << status.ErrorMessage(); } - // This method should be called from within Initialize() only and before the creation of the session state. - // This ensures all providers have been registered in the session and the session state is consistent with the providers. - void InferenceSession::UpdateProvidersWithSharedAllocators() { - using namespace std; - const auto& provider_ids = execution_providers_.GetIds(); - for (const auto& one_shared_alloc : environment_.GetRegisteredSharedAllocators()) { - for (const auto& id : provider_ids) { - auto* provider_ptr = execution_providers_.Get(id); - provider_ptr->ReplaceAllocator(one_shared_alloc); + if (session_profiler_.IsEnabled()) { + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "session_initialization", tp); + } + + if (status.IsOK()) { + for (auto& xp : execution_providers_) { + auto end_status = xp->OnSessionInitializationEnd(); + if (status.IsOK()) { + status = end_status; } } } - int InferenceSession::GetCurrentNumRuns() const { - return current_num_runs_.load(); - } + return status; +} - const std::vector& InferenceSession::GetRegisteredProviderTypes() const { - return execution_providers_.GetIds(); - } - - const ProviderOptionsMap& InferenceSession::GetAllProviderOptions() const { - return execution_providers_.GetAllProviderOptions(); - } - - const SessionOptions& InferenceSession::GetSessionOptions() const { - return session_options_; - } - - const DataTransferManager& InferenceSession::GetDataTransferManager() const { - return data_transfer_mgr_; - } - - common::Status InferenceSession::CheckShapes(const std::string& input_name, const TensorShape& input_shape, - const TensorShape& expected_shape) const { - auto input_shape_sz = input_shape.NumDimensions(); - auto expected_shape_sz = expected_shape.NumDimensions(); - if (input_shape_sz != expected_shape_sz) { - std::ostringstream ostr; - ostr << "Invalid rank for input: " << input_name << " Got: " << input_shape_sz << " Expected: " << expected_shape_sz - << " Please fix either the inputs or the model."; - return Status(ONNXRUNTIME, INVALID_ARGUMENT, ostr.str()); +// This method should be called from within Initialize() only and before the creation of the session state. +// This ensures all providers have been registered in the session and the session state is consistent with the providers. +void InferenceSession::UpdateProvidersWithSharedAllocators() { + using namespace std; + const auto& provider_ids = execution_providers_.GetIds(); + for (const auto& one_shared_alloc : environment_.GetRegisteredSharedAllocators()) { + for (const auto& id : provider_ids) { + auto* provider_ptr = execution_providers_.Get(id); + provider_ptr->ReplaceAllocator(one_shared_alloc); } + } +} - std::vector invalid_dim_indices; - for (size_t i = 0; i < input_shape_sz; ++i) { - if (expected_shape[i] < 0) { - continue; // this represents a symbolic shape dimension - } - if (input_shape[i] != expected_shape[i]) { - invalid_dim_indices.push_back(i); - } +int InferenceSession::GetCurrentNumRuns() const { + return current_num_runs_.load(); +} + +const std::vector& InferenceSession::GetRegisteredProviderTypes() const { + return execution_providers_.GetIds(); +} + +const ProviderOptionsMap& InferenceSession::GetAllProviderOptions() const { + return execution_providers_.GetAllProviderOptions(); +} + +const SessionOptions& InferenceSession::GetSessionOptions() const { + return session_options_; +} + +const DataTransferManager& InferenceSession::GetDataTransferManager() const { + return data_transfer_mgr_; +} + +common::Status InferenceSession::CheckShapes(const std::string& input_name, const TensorShape& input_shape, + const TensorShape& expected_shape) const { + auto input_shape_sz = input_shape.NumDimensions(); + auto expected_shape_sz = expected_shape.NumDimensions(); + if (input_shape_sz != expected_shape_sz) { + std::ostringstream ostr; + ostr << "Invalid rank for input: " << input_name << " Got: " << input_shape_sz << " Expected: " << expected_shape_sz + << " Please fix either the inputs or the model."; + return Status(ONNXRUNTIME, INVALID_ARGUMENT, ostr.str()); + } + + std::vector invalid_dim_indices; + for (size_t i = 0; i < input_shape_sz; ++i) { + if (expected_shape[i] < 0) { + continue; // this represents a symbolic shape dimension } - - if (!invalid_dim_indices.empty()) { - std::ostringstream ostr; - ostr << "Got invalid dimensions for input: " << input_name << " for the following indices\n"; - for (size_t i = 0, end = invalid_dim_indices.size(); i < end; ++i) { - size_t idx = invalid_dim_indices[i]; - ostr << " index: " << idx << " Got: " << input_shape[idx] << " Expected: " << expected_shape[idx] << "\n"; - } - ostr << " Please fix either the inputs or the model."; - return Status(ONNXRUNTIME, INVALID_ARGUMENT, ostr.str()); + if (input_shape[i] != expected_shape[i]) { + invalid_dim_indices.push_back(i); } + } + if (!invalid_dim_indices.empty()) { + std::ostringstream ostr; + ostr << "Got invalid dimensions for input: " << input_name << " for the following indices\n"; + for (size_t i = 0, end = invalid_dim_indices.size(); i < end; ++i) { + size_t idx = invalid_dim_indices[i]; + ostr << " index: " << idx << " Got: " << input_shape[idx] << " Expected: " << expected_shape[idx] << "\n"; + } + ostr << " Please fix either the inputs or the model."; + return Status(ONNXRUNTIME, INVALID_ARGUMENT, ostr.str()); + } + + return Status::OK(); +} + +static common::Status CheckTypes(MLDataType actual, MLDataType expected) { + if (actual == expected) { return Status::OK(); } - - static common::Status CheckTypes(MLDataType actual, MLDataType expected) { - if (actual == expected) { - return Status::OK(); - } #ifdef ORT_NO_RTTI - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Unexpected input data type"); + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Unexpected input data type"); #else auto actual_name = std::string(typeid(*actual).name()); auto expected_name = std::string(typeid(*expected).name()); return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Unexpected input data type. Actual: (" + actual_name + ") , expected: (" + expected_name + ")"); #endif +} + +common::Status InferenceSession::ValidateInputs(const std::vector& feed_names, + const std::vector& feeds) const { + if (feed_names.size() != feeds.size()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Size mismatch: feed_names has ", feed_names.size(), + "elements, but feeds has ", feeds.size(), " elements."); } - common::Status InferenceSession::ValidateInputs(const std::vector& feed_names, - const std::vector& feeds) const { - if (feed_names.size() != feeds.size()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Size mismatch: feed_names has ", feed_names.size(), - "elements, but feeds has ", feeds.size(), " elements."); + for (size_t i = 0; i < feeds.size(); ++i) { + const auto& feed_name = feed_names[i]; + + auto iter = input_def_map_.find(feed_name); + if (input_def_map_.end() == iter) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid Feed Input Name:", feed_name); } - for (size_t i = 0; i < feeds.size(); ++i) { - const auto& feed_name = feed_names[i]; - - auto iter = input_def_map_.find(feed_name); - if (input_def_map_.end() == iter) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid Feed Input Name:", feed_name); + auto expected_type = iter->second.ml_data_type; + auto& input_ml_value = feeds.at(i); + if (input_ml_value.IsTensor()) { + // check for type + if (!expected_type->IsTensorType()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, + " is not expected to be of type tensor."); } + auto expected_element_type = expected_type->AsTensorType()->GetElementType(); + auto input_element_type = input_ml_value.Get().DataType(); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); - auto expected_type = iter->second.ml_data_type; - auto& input_ml_value = feeds.at(i); - if (input_ml_value.IsTensor()) { - // check for type - if (!expected_type->IsTensorType()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, - " is not expected to be of type tensor."); - } - auto expected_element_type = expected_type->AsTensorType()->GetElementType(); - auto input_element_type = input_ml_value.Get().DataType(); - ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); - - // check for shape - const auto& expected_shape = iter->second.tensor_shape; - if (expected_shape.NumDimensions() > 0) { - const auto& input_shape = input_ml_value.Get().Shape(); - ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape)); - } - } else if (input_ml_value.IsSparseTensor()) { + // check for shape + const auto& expected_shape = iter->second.tensor_shape; + if (expected_shape.NumDimensions() > 0) { + const auto& input_shape = input_ml_value.Get().Shape(); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckShapes(feed_name, input_shape, expected_shape)); + } + } else if (input_ml_value.IsSparseTensor()) { #if !defined(ORT_MINIMAL_BUILD) - if (!expected_type->IsSparseTensorType()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, - " is not expected to be of type sparse tensor."); - } - auto expected_element_type = expected_type->AsSparseTensorType()->GetElementType(); - auto input_element_type = input_ml_value.Get().Values().DataType(); - // TODO: In the future, when sparsetensors are in use, find out how to properly verify the shape - ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); + if (!expected_type->IsSparseTensorType()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, + " is not expected to be of type sparse tensor."); + } + auto expected_element_type = expected_type->AsSparseTensorType()->GetElementType(); + auto input_element_type = input_ml_value.Get().Values().DataType(); + // TODO: In the future, when sparsetensors are in use, find out how to properly verify the shape + ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); #else return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name ", feed_name, " is a sparse tensor, which is not supported in this build."); #endif - } else if (input_ml_value.IsTensorSequence()) { - if (!expected_type->IsTensorSequenceType()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, - " is not expected to be of type tensor sequence."); - } - auto expected_element_type = expected_type->AsSequenceTensorBase()->GetElementType(); - auto input_element_type = input_ml_value.Get().DataType(); - ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); - } else { - auto input_type = input_ml_value.Type(); - ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_type, expected_type)); + } else if (input_ml_value.IsTensorSequence()) { + if (!expected_type->IsTensorSequenceType()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input with name: ", feed_name, + " is not expected to be of type tensor sequence."); } + auto expected_element_type = expected_type->AsSequenceTensorBase()->GetElementType(); + auto input_element_type = input_ml_value.Get().DataType(); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_element_type, expected_element_type)); + } else { + auto input_type = input_ml_value.Type(); + ORT_RETURN_IF_ERROR_SESSIONID_(CheckTypes(input_type, expected_type)); } - - return Status::OK(); } - common::Status InferenceSession::ValidateOutputs(const std::vector& output_names, - const std::vector* p_fetches) const { - if (p_fetches == nullptr) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Output vector pointer is NULL"); - } + return Status::OK(); +} - if (output_names.empty()) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "At least one output should be requested."); - } - - if (!p_fetches->empty() && (output_names.size() != p_fetches->size())) { - std::ostringstream ostr; - ostr << "Output vector incorrectly sized: output_names.size(): " << output_names.size() - << "p_fetches->size(): " << p_fetches->size(); - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); - } - - for (const auto& name : output_names) { - if (model_output_names_.find(name) == model_output_names_.end()) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid Output Name:" + name); - } - } - - // TODO add more validation here like checking shape of the allocated buffers - - return common::Status::OK(); +common::Status InferenceSession::ValidateOutputs(const std::vector& output_names, + const std::vector* p_fetches) const { + if (p_fetches == nullptr) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Output vector pointer is NULL"); } - Status InferenceSession::Run(const RunOptions& run_options, - const std::vector& feed_names, const std::vector& feeds, - const std::vector& output_names, std::vector* p_fetches, - const std::vector* p_fetches_device_info) { - TimePoint tp; - if (session_profiler_.IsEnabled()) { - tp = session_profiler_.StartTime(); + if (output_names.empty()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "At least one output should be requested."); + } + + if (!p_fetches->empty() && (output_names.size() != p_fetches->size())) { + std::ostringstream ostr; + ostr << "Output vector incorrectly sized: output_names.size(): " << output_names.size() + << "p_fetches->size(): " << p_fetches->size(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + + for (const auto& name : output_names) { + if (model_output_names_.find(name) == model_output_names_.end()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid Output Name:" + name); } + } + + // TODO add more validation here like checking shape of the allocated buffers + + return common::Status::OK(); +} + +Status InferenceSession::Run(const RunOptions& run_options, + const std::vector& feed_names, const std::vector& feeds, + const std::vector& output_names, std::vector* p_fetches, + const std::vector* p_fetches_device_info) { + TimePoint tp; + if (session_profiler_.IsEnabled()) { + tp = session_profiler_.StartTime(); + } #ifdef ONNXRUNTIME_ENABLE_INSTRUMENT - TraceLoggingActivity ortrun_activity; - ortrun_activity.SetRelatedActivity(session_activity); - TraceLoggingWriteStart(ortrun_activity, "OrtRun"); + TraceLoggingActivity ortrun_activity; + ortrun_activity.SetRelatedActivity(session_activity); + TraceLoggingWriteStart(ortrun_activity, "OrtRun"); #endif - Status retval = Status::OK(); - const Env& env = Env::Default(); + Status retval = Status::OK(); + const Env& env = Env::Default(); - std::vector exec_providers_to_stop; - exec_providers_to_stop.reserve(execution_providers_.NumProviders()); + std::vector exec_providers_to_stop; + exec_providers_to_stop.reserve(execution_providers_.NumProviders()); - ORT_TRY { - if (!is_inited_) { - LOGS(*session_logger_, ERROR) << "Session was not initialized"; - return Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); + ORT_TRY { + if (!is_inited_) { + LOGS(*session_logger_, ERROR) << "Session was not initialized"; + return Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); + } + + // log evaluation start to trace logging provider + env.GetTelemetryProvider().LogEvaluationStart(); + + ORT_RETURN_IF_ERROR_SESSIONID_(ValidateInputs(feed_names, feeds)); + ORT_RETURN_IF_ERROR_SESSIONID_(ValidateOutputs(output_names, p_fetches)); + + FeedsFetchesInfo info(feed_names, output_names, session_state_->GetOrtValueNameIdxMap()); + FeedsFetchesManager feeds_fetches_manager{std::move(info)}; + + if (p_fetches_device_info) { + // populate the target device info. ignored if pre-allocated fetches are provided + const auto& fetch_device_info = *p_fetches_device_info; + auto& fetch_info = feeds_fetches_manager.GetMutableFetchesDeviceCopyInfo(); + + for (size_t i = 0, end = output_names.size(); i < end; ++i) { + fetch_info[i].target_device = fetch_device_info[i]; } + } - // log evaluation start to trace logging provider - env.GetTelemetryProvider().LogEvaluationStart(); + if (!run_options.run_tag.empty()) { + LOGS(*session_logger_, INFO) << "Running with tag: " << run_options.run_tag; + } - ORT_RETURN_IF_ERROR_SESSIONID_(ValidateInputs(feed_names, feeds)); - ORT_RETURN_IF_ERROR_SESSIONID_(ValidateOutputs(output_names, p_fetches)); + ++current_num_runs_; - FeedsFetchesInfo info(feed_names, output_names, session_state_->GetOrtValueNameIdxMap()); - FeedsFetchesManager feeds_fetches_manager{std::move(info)}; + // TODO should we add this exec to the list of executors? i guess its not needed now? - if (p_fetches_device_info) { - // populate the target device info. ignored if pre-allocated fetches are provided - const auto& fetch_device_info = *p_fetches_device_info; - auto& fetch_info = feeds_fetches_manager.GetMutableFetchesDeviceCopyInfo(); + // scope of owned_run_logger is just the call to Execute. + // If Execute ever becomes async we need a different approach + std::unique_ptr owned_run_logger; + auto run_logger = CreateLoggerForRun(run_options, owned_run_logger); - for (size_t i = 0, end = output_names.size(); i < end; ++i) { - fetch_info[i].target_device = fetch_device_info[i]; - } - } + // info all execution providers InferenceSession:Run started + // TODO: only call OnRunStart for all providers in-use + for (auto& xp : execution_providers_) { + // call OnRunStart and add to exec_providers_to_stop if successful + auto start_func = [&xp, &exec_providers_to_stop]() { + auto status = xp->OnRunStart(); + if (status.IsOK()) + exec_providers_to_stop.push_back(xp.get()); - if (!run_options.run_tag.empty()) { - LOGS(*session_logger_, INFO) << "Running with tag: " << run_options.run_tag; - } + return status; + }; - ++current_num_runs_; - - // TODO should we add this exec to the list of executors? i guess its not needed now? - - // scope of owned_run_logger is just the call to Execute. - // If Execute ever becomes async we need a different approach - std::unique_ptr owned_run_logger; - auto run_logger = CreateLoggerForRun(run_options, owned_run_logger); - - // info all execution providers InferenceSession:Run started - // TODO: only call OnRunStart for all providers in-use - for (auto& xp : execution_providers_) { - // call OnRunStart and add to exec_providers_to_stop if successful - auto start_func = [&xp, &exec_providers_to_stop]() { - auto status = xp->OnRunStart(); - if (status.IsOK()) - exec_providers_to_stop.push_back(xp.get()); - - return status; - }; - - ORT_CHECK_AND_SET_RETVAL(start_func()); - } + ORT_CHECK_AND_SET_RETVAL(start_func()); + } #if !defined(ORT_MINIMAL_BUILD) - if (run_options.only_execute_path_to_fetches) { - session_state_->UpdateToBeExecutedNodes(feeds_fetches_manager.GetFeedsFetchesInfo().fetches_mlvalue_idxs); - } + if (run_options.only_execute_path_to_fetches) { + session_state_->UpdateToBeExecutedNodes(feeds_fetches_manager.GetFeedsFetchesInfo().fetches_mlvalue_idxs); + } #endif - // execute the graph - ORT_CHECK_AND_SET_RETVAL(utils::ExecuteGraph(*session_state_, feeds_fetches_manager, feeds, *p_fetches, - session_options_.execution_mode, run_options.terminate, run_logger, - run_options.only_execute_path_to_fetches)); - } - ORT_CATCH(const std::exception& e) { - ORT_HANDLE_EXCEPTION([&]() { - retval = Status(common::ONNXRUNTIME, common::FAIL, e.what()); - }); - } - ORT_CATCH(...) { - retval = Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Run()"); - } + // execute the graph + ORT_CHECK_AND_SET_RETVAL(utils::ExecuteGraph(*session_state_, feeds_fetches_manager, feeds, *p_fetches, + session_options_.execution_mode, run_options.terminate, run_logger, + run_options.only_execute_path_to_fetches)); + } + ORT_CATCH(const std::exception& e) { + ORT_HANDLE_EXCEPTION([&]() { + retval = Status(common::ONNXRUNTIME, common::FAIL, e.what()); + }); + } + ORT_CATCH(...) { + retval = Status(common::ONNXRUNTIME, common::RUNTIME_EXCEPTION, "Encountered unknown exception in Run()"); + } - // info all execution providers InferenceSession:Run ended - for (auto* xp : exec_providers_to_stop) { - auto status = xp->OnRunEnd(); - ORT_CHECK_AND_SET_RETVAL(status); - } + // info all execution providers InferenceSession:Run ended + for (auto* xp : exec_providers_to_stop) { + auto status = xp->OnRunEnd(); + ORT_CHECK_AND_SET_RETVAL(status); + } - --current_num_runs_; + --current_num_runs_; - // keep track of telemetry - ++telemetry_.total_runs_since_last_; - telemetry_.total_run_duration_since_last_ += TimeDiffMicroSeconds(tp); + // keep track of telemetry + ++telemetry_.total_runs_since_last_; + telemetry_.total_run_duration_since_last_ += TimeDiffMicroSeconds(tp); - // time to send telemetry? - if (TimeDiffMicroSeconds(telemetry_.time_sent_last_) > telemetry_.kDurationBetweenSending) { - // send the telemetry - env.GetTelemetryProvider().LogRuntimePerf(session_id_, telemetry_.total_runs_since_last_, - telemetry_.total_run_duration_since_last_); - // reset counters - telemetry_.time_sent_last_ = std::chrono::high_resolution_clock::now(); - telemetry_.total_runs_since_last_ = 0; - telemetry_.total_run_duration_since_last_ = 0; - } + // time to send telemetry? + if (TimeDiffMicroSeconds(telemetry_.time_sent_last_) > telemetry_.kDurationBetweenSending) { + // send the telemetry + env.GetTelemetryProvider().LogRuntimePerf(session_id_, telemetry_.total_runs_since_last_, + telemetry_.total_run_duration_since_last_); + // reset counters + telemetry_.time_sent_last_ = std::chrono::high_resolution_clock::now(); + telemetry_.total_runs_since_last_ = 0; + telemetry_.total_run_duration_since_last_ = 0; + } - // log evaluation stop to trace logging provider - env.GetTelemetryProvider().LogEvaluationStop(); + // log evaluation stop to trace logging provider + env.GetTelemetryProvider().LogEvaluationStop(); - // send out profiling events (optional) - if (session_profiler_.IsEnabled()) { - session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_run", tp); - } + // send out profiling events (optional) + if (session_profiler_.IsEnabled()) { + session_profiler_.EndTimeAndRecordEvent(profiling::SESSION_EVENT, "model_run", tp); + } #ifdef ONNXRUNTIME_ENABLE_INSTRUMENT - TraceLoggingWriteStop(ortrun_activity, "OrtRun"); + TraceLoggingWriteStop(ortrun_activity, "OrtRun"); #endif - return retval; + return retval; +} + +common::Status InferenceSession::Run(const NameMLValMap& feeds, const std::vector& output_names, + std::vector* p_fetches) { + return Run(RunOptions(), feeds, output_names, p_fetches); +} + +common::Status InferenceSession::Run(const RunOptions& run_options, const NameMLValMap& feeds_map, + const std::vector& output_names, std::vector* p_fetches) { + std::vector feed_names; + std::vector feeds; + + auto num_feeds = feeds_map.size(); + feed_names.reserve(num_feeds); + feeds.reserve(num_feeds); + + for (auto& pair : feeds_map) { + feed_names.push_back(pair.first); + feeds.push_back(pair.second); } - common::Status InferenceSession::Run(const NameMLValMap& feeds, const std::vector& output_names, - std::vector* p_fetches) { - return Run(RunOptions(), feeds, output_names, p_fetches); - } + return Run(run_options, feed_names, feeds, output_names, p_fetches, nullptr); +} - common::Status InferenceSession::Run(const RunOptions& run_options, const NameMLValMap& feeds_map, - const std::vector& output_names, std::vector* p_fetches) { - std::vector feed_names; - std::vector feeds; - - auto num_feeds = feeds_map.size(); - feed_names.reserve(num_feeds); - feeds.reserve(num_feeds); - - for (auto& pair : feeds_map) { - feed_names.push_back(pair.first); - feeds.push_back(pair.second); +std::pair InferenceSession::GetModelMetadata() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); } - - return Run(run_options, feed_names, feeds, output_names, p_fetches, nullptr); } - std::pair InferenceSession::GetModelMetadata() const { - { - std::lock_guard l(session_mutex_); - if (!is_model_loaded_) { - LOGS(*session_logger_, ERROR) << "Model was not loaded"; - return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); - } + return std::make_pair(common::Status::OK(), &model_metadata_); +} + +std::pair InferenceSession::GetModelInputs() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); } - - return std::make_pair(common::Status::OK(), &model_metadata_); } - std::pair InferenceSession::GetModelInputs() const { - { - std::lock_guard l(session_mutex_); - if (!is_model_loaded_) { - LOGS(*session_logger_, ERROR) << "Model was not loaded"; - return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); - } + // return required inputs (excludes any inputs used for overriding initializers) + return std::make_pair(common::Status::OK(), &model_->MainGraph().GetInputs()); +} + +std::pair InferenceSession::GetOverridableInitializers() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); } - - // return required inputs (excludes any inputs used for overriding initializers) - return std::make_pair(common::Status::OK(), &model_->MainGraph().GetInputs()); } - std::pair InferenceSession::GetOverridableInitializers() const { - { - std::lock_guard l(session_mutex_); - if (!is_model_loaded_) { - LOGS(*session_logger_, ERROR) << "Model was not loaded"; - return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); - } + // returns a list of initializers that can be overriden. + return std::make_pair(common::Status::OK(), &model_->MainGraph().GetOverridableInitializers()); +} + +std::pair InferenceSession::GetModelOutputs() const { + { + std::lock_guard l(session_mutex_); + if (!is_model_loaded_) { + LOGS(*session_logger_, ERROR) << "Model was not loaded"; + return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); } - - // returns a list of initializers that can be overriden. - return std::make_pair(common::Status::OK(), &model_->MainGraph().GetOverridableInitializers()); } - std::pair InferenceSession::GetModelOutputs() const { - { - std::lock_guard l(session_mutex_); - if (!is_model_loaded_) { - LOGS(*session_logger_, ERROR) << "Model was not loaded"; - return std::make_pair(common::Status(common::ONNXRUNTIME, common::FAIL, "Model was not loaded."), nullptr); - } + return std::make_pair(common::Status::OK(), &output_def_list_); +} + +common::Status InferenceSession::NewIOBinding(std::unique_ptr* io_binding) { + { + std::lock_guard l(session_mutex_); + if (!is_inited_) { + LOGS(*session_logger_, ERROR) << "Session was not initialized"; + return common::Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); } - - return std::make_pair(common::Status::OK(), &output_def_list_); } - common::Status InferenceSession::NewIOBinding(std::unique_ptr* io_binding) { - { - std::lock_guard l(session_mutex_); - if (!is_inited_) { - LOGS(*session_logger_, ERROR) << "Session was not initialized"; - return common::Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); - } - } + // private constructor, can't use make_unique + *io_binding = std::unique_ptr(new IOBinding(*session_state_)); + return Status::OK(); +} - // private constructor, can't use make_unique - *io_binding = std::unique_ptr(new IOBinding(*session_state_)); - return Status::OK(); - } +common::Status InferenceSession::Run(const RunOptions& run_options, IOBinding& io_binding) { + // TODO should Run() call io_binding.SynchronizeInputs() or should it let the callers do it? + // io_binding.SynchronizeInputs(); + return Run(run_options, io_binding.GetInputNames(), io_binding.GetInputs(), io_binding.GetOutputNames(), + &io_binding.GetOutputs(), &io_binding.GetOutputsDeviceInfo()); +} - common::Status InferenceSession::Run(const RunOptions& run_options, IOBinding& io_binding) { - // TODO should Run() call io_binding.SynchronizeInputs() or should it let the callers do it? - // io_binding.SynchronizeInputs(); - return Run(run_options, io_binding.GetInputNames(), io_binding.GetInputs(), io_binding.GetOutputNames(), - &io_binding.GetOutputs(), &io_binding.GetOutputsDeviceInfo()); - } +common::Status InferenceSession::Run(IOBinding& io_binding) { + RunOptions run_options; + return Run(run_options, io_binding); +} - common::Status InferenceSession::Run(IOBinding& io_binding) { - RunOptions run_options; - return Run(run_options, io_binding); - } +template +void InferenceSession::StartProfiling(const std::basic_string& file_prefix) { + std::basic_ostringstream ss; + ss << file_prefix << "_" << GetCurrentTimeString() << ".json"; + session_profiler_.StartProfiling(ss.str()); +} - template - void InferenceSession::StartProfiling(const std::basic_string& file_prefix) { - std::basic_ostringstream ss; - ss << file_prefix << "_" << GetCurrentTimeString() << ".json"; - session_profiler_.StartProfiling(ss.str()); - } - - void InferenceSession::StartProfiling(const std::string& file_prefix) { - StartProfiling(file_prefix); - } +void InferenceSession::StartProfiling(const std::string& file_prefix) { + StartProfiling(file_prefix); +} #ifdef _WIN32 - void InferenceSession::StartProfiling(const std::wstring& file_prefix) { - StartProfiling(file_prefix); - } +void InferenceSession::StartProfiling(const std::wstring& file_prefix) { + StartProfiling(file_prefix); +} #endif - void InferenceSession::StartProfiling(const logging::Logger* logger_ptr) { - session_profiler_.StartProfiling(logger_ptr); - } +void InferenceSession::StartProfiling(const logging::Logger* logger_ptr) { + session_profiler_.StartProfiling(logger_ptr); +} - std::string InferenceSession::EndProfiling() { - if (is_model_loaded_) { - if (session_profiler_.IsEnabled()) { - return session_profiler_.EndProfiling(); - } else { - LOGS(*session_logger_, VERBOSE) << "Profiler is disabled."; - return std::string(); - } +std::string InferenceSession::EndProfiling() { + if (is_model_loaded_) { + if (session_profiler_.IsEnabled()) { + return session_profiler_.EndProfiling(); + } else { + LOGS(*session_logger_, VERBOSE) << "Profiler is disabled."; + return std::string(); } - LOGS(*session_logger_, ERROR) << "Could not write a profile because no model was loaded."; - return std::string(); } + LOGS(*session_logger_, ERROR) << "Could not write a profile because no model was loaded."; + return std::string(); +} - const profiling::Profiler& InferenceSession::GetProfiling() const { - return session_profiler_; - } +const profiling::Profiler& InferenceSession::GetProfiling() const { + return session_profiler_; +} - AllocatorPtr InferenceSession::GetAllocator(const OrtMemoryInfo& mem_info) const { - return session_state_->GetAllocator(mem_info); - } +AllocatorPtr InferenceSession::GetAllocator(const OrtMemoryInfo& mem_info) const { + return session_state_->GetAllocator(mem_info); +} #if !defined(ORT_MINIMAL_BUILD) - // assumes model has already been loaded before - common::Status InferenceSession::DoPostLoadProcessing(onnxruntime::Model& model) { - // TODO add other post load processing here - common::Status status = SaveModelMetadata(model); - return status; - } +// assumes model has already been loaded before +common::Status InferenceSession::DoPostLoadProcessing(onnxruntime::Model& model) { + // TODO add other post load processing here + common::Status status = SaveModelMetadata(model); + return status; +} #endif - common::Status InferenceSession::SaveModelMetadata(const onnxruntime::Model& model) { - VLOGS(*session_logger_, 1) << "Saving model metadata"; - const onnxruntime::Graph& graph = model.MainGraph(); +common::Status InferenceSession::SaveModelMetadata(const onnxruntime::Model& model) { + VLOGS(*session_logger_, 1) << "Saving model metadata"; + const onnxruntime::Graph& graph = model.MainGraph(); - // save model metadata - model_metadata_.producer_name = model.ProducerName(); - model_metadata_.description = model.DocString(); - model_metadata_.domain = model.Domain(); - model_metadata_.version = model.ModelVersion(); - model_metadata_.custom_metadata_map = model.MetaData(); - model_metadata_.graph_name = graph.Name(); + // save model metadata + model_metadata_.producer_name = model.ProducerName(); + model_metadata_.description = model.DocString(); + model_metadata_.domain = model.Domain(); + model_metadata_.version = model.ModelVersion(); + model_metadata_.custom_metadata_map = model.MetaData(); + model_metadata_.graph_name = graph.Name(); - required_inputs_.clear(); - for (auto input : graph.GetInputs()) { - required_inputs_.insert(input->Name()); - } - - auto add_inputs = [this](const InputDefList& inputs) { - input_def_map_.clear(); - input_def_map_.reserve(inputs.size()); - for (auto elem : inputs) { - auto elem_type = utils::GetMLDataType(*elem); - auto elem_shape_proto = elem->Shape(); - input_def_map_.insert( - {elem->Name(), - InputDefMetaData( - elem, elem_type, - elem_shape_proto ? utils::GetTensorShapeFromTensorShapeProto(*elem_shape_proto) : TensorShape())}); - } - }; - - if (graph.CanOverrideInitializer()) { - // for IR 4 or higher it is optional to have a matching graph input for an initializer, and if one exists the - // initializer is explicitly overridable. - add_inputs(graph.GetInputsIncludingInitializers()); - } else { - // for IR < 4 we don't allow overriding initializers so that they can be treated as constant. exclude them from - // the list of valid inputs by just using the GetInputs() list. - add_inputs(graph.GetInputs()); - } - - // save outputs - const auto& outputs = graph.GetOutputs(); - output_def_list_ = outputs; // A direct copy of outputs - - model_output_names_.clear(); - model_output_names_.reserve(outputs.size()); - for (const auto& elem : outputs) { - model_output_names_.insert(elem->Name()); - } - - VLOGS(*session_logger_, 1) << "Done saving model metadata"; - return common::Status::OK(); + required_inputs_.clear(); + for (auto input : graph.GetInputs()) { + required_inputs_.insert(input->Name()); } - // Create a Logger for a single execution if possible. Otherwise use the default logger. - // If a new logger is created, it will also be stored in new_run_logger, - // which must remain valid for the duration of the execution. - // If the default logger is used, new_run_logger will remain empty. - // The returned value should be used in the execution. - const logging::Logger& InferenceSession::CreateLoggerForRun(const RunOptions& run_options, - std::unique_ptr& new_run_logger) { - const logging::Logger* run_logger; - - // create a per-run logger if we can - if (logging_manager_ != nullptr) { - std::string run_log_id{session_options_.session_logid}; - - if (!session_options_.session_logid.empty() && !run_options.run_tag.empty()) { - run_log_id += ":"; - } - - run_log_id += run_options.run_tag; - - logging::Severity severity = logging::Severity::kWARNING; - if (run_options.run_log_severity_level == -1) { - severity = session_logger_->GetSeverity(); - } else { - ORT_ENFORCE(run_options.run_log_severity_level >= 0 && - run_options.run_log_severity_level <= static_cast(logging::Severity::kFATAL), - "Invalid run log severity level. Not a valid onnxruntime::logging::Severity value: ", - run_options.run_log_severity_level); - severity = static_cast(run_options.run_log_severity_level); - } - - new_run_logger = logging_manager_->CreateLogger(run_log_id, severity, false, run_options.run_log_verbosity_level); - - run_logger = new_run_logger.get(); - VLOGS(*run_logger, 1) << "Created logger for run with id of " << run_log_id; - } else { - // fallback to using default logger. this does NOT have any session or run specific id/tag in it - run_logger = session_logger_; - VLOGS(*run_logger, 1) << "Using default logger for run " << run_options.run_tag; + auto add_inputs = [this](const InputDefList& inputs) { + input_def_map_.clear(); + input_def_map_.reserve(inputs.size()); + for (auto elem : inputs) { + auto elem_type = utils::GetMLDataType(*elem); + auto elem_shape_proto = elem->Shape(); + input_def_map_.insert( + {elem->Name(), + InputDefMetaData( + elem, elem_type, + elem_shape_proto ? utils::GetTensorShapeFromTensorShapeProto(*elem_shape_proto) : TensorShape())}); } + }; - return *run_logger; + if (graph.CanOverrideInitializer()) { + // for IR 4 or higher it is optional to have a matching graph input for an initializer, and if one exists the + // initializer is explicitly overridable. + add_inputs(graph.GetInputsIncludingInitializers()); + } else { + // for IR < 4 we don't allow overriding initializers so that they can be treated as constant. exclude them from + // the list of valid inputs by just using the GetInputs() list. + add_inputs(graph.GetInputs()); } - void InferenceSession::InitLogger(logging::LoggingManager* logging_manager) { - // create logger for session, using provided logging manager if possible - if (logging_manager != nullptr) { - logging::Severity severity = logging::Severity::kWARNING; - if (session_options_.session_log_severity_level == -1) { - severity = logging::LoggingManager::DefaultLogger().GetSeverity(); - } else { - ORT_ENFORCE(session_options_.session_log_severity_level >= 0 && - session_options_.session_log_severity_level <= static_cast(logging::Severity::kFATAL), - "Invalid session log severity level. Not a valid onnxruntime::logging::Severity value: ", - session_options_.session_log_severity_level); - severity = static_cast(session_options_.session_log_severity_level); - } + // save outputs + const auto& outputs = graph.GetOutputs(); + output_def_list_ = outputs; // A direct copy of outputs - owned_session_logger_ = logging_manager_->CreateLogger(session_options_.session_logid, severity, false, - session_options_.session_log_verbosity_level); - session_logger_ = owned_session_logger_.get(); - } else { - session_logger_ = &logging::LoggingManager::DefaultLogger(); - } + model_output_names_.clear(); + model_output_names_.reserve(outputs.size()); + for (const auto& elem : outputs) { + model_output_names_.insert(elem->Name()); } + VLOGS(*session_logger_, 1) << "Done saving model metadata"; + return common::Status::OK(); +} + +// Create a Logger for a single execution if possible. Otherwise use the default logger. +// If a new logger is created, it will also be stored in new_run_logger, +// which must remain valid for the duration of the execution. +// If the default logger is used, new_run_logger will remain empty. +// The returned value should be used in the execution. +const logging::Logger& InferenceSession::CreateLoggerForRun(const RunOptions& run_options, + std::unique_ptr& new_run_logger) { + const logging::Logger* run_logger; + + // create a per-run logger if we can + if (logging_manager_ != nullptr) { + std::string run_log_id{session_options_.session_logid}; + + if (!session_options_.session_logid.empty() && !run_options.run_tag.empty()) { + run_log_id += ":"; + } + + run_log_id += run_options.run_tag; + + logging::Severity severity = logging::Severity::kWARNING; + if (run_options.run_log_severity_level == -1) { + severity = session_logger_->GetSeverity(); + } else { + ORT_ENFORCE(run_options.run_log_severity_level >= 0 && + run_options.run_log_severity_level <= static_cast(logging::Severity::kFATAL), + "Invalid run log severity level. Not a valid onnxruntime::logging::Severity value: ", + run_options.run_log_severity_level); + severity = static_cast(run_options.run_log_severity_level); + } + + new_run_logger = logging_manager_->CreateLogger(run_log_id, severity, false, run_options.run_log_verbosity_level); + + run_logger = new_run_logger.get(); + VLOGS(*run_logger, 1) << "Created logger for run with id of " << run_log_id; + } else { + // fallback to using default logger. this does NOT have any session or run specific id/tag in it + run_logger = session_logger_; + VLOGS(*run_logger, 1) << "Using default logger for run " << run_options.run_tag; + } + + return *run_logger; +} + +void InferenceSession::InitLogger(logging::LoggingManager* logging_manager) { + // create logger for session, using provided logging manager if possible + if (logging_manager != nullptr) { + logging::Severity severity = logging::Severity::kWARNING; + if (session_options_.session_log_severity_level == -1) { + severity = logging::LoggingManager::DefaultLogger().GetSeverity(); + } else { + ORT_ENFORCE(session_options_.session_log_severity_level >= 0 && + session_options_.session_log_severity_level <= static_cast(logging::Severity::kFATAL), + "Invalid session log severity level. Not a valid onnxruntime::logging::Severity value: ", + session_options_.session_log_severity_level); + severity = static_cast(session_options_.session_log_severity_level); + } + + owned_session_logger_ = logging_manager_->CreateLogger(session_options_.session_logid, severity, false, + session_options_.session_log_verbosity_level); + session_logger_ = owned_session_logger_.get(); + } else { + session_logger_ = &logging::LoggingManager::DefaultLogger(); + } +} + #if !defined(ORT_MINIMAL_BUILD) - // Registers all the predefined transformers with transformer manager - void InferenceSession::AddPredefinedTransformers(GraphTransformerManager& transformer_manager, - TransformerLevel graph_optimization_level, - const std::vector& custom_list) { - auto add_transformers = [&](TransformerLevel level) { - // Generate and register transformers for level - auto transformers_to_register = - optimizer_utils::GenerateTransformers(level, session_options_.free_dimension_overrides, - *execution_providers_.Get(onnxruntime::kCpuExecutionProvider), - custom_list); - for (auto& entry : transformers_to_register) { - transformer_manager.Register(std::move(entry), level); - } - }; +// Registers all the predefined transformers with transformer manager +void InferenceSession::AddPredefinedTransformers(GraphTransformerManager& transformer_manager, + TransformerLevel graph_optimization_level, + const std::vector& custom_list) { + auto add_transformers = [&](TransformerLevel level) { + // Generate and register transformers for level + auto transformers_to_register = + optimizer_utils::GenerateTransformers(level, session_options_.free_dimension_overrides, + *execution_providers_.Get(onnxruntime::kCpuExecutionProvider), + custom_list); + for (auto& entry : transformers_to_register) { + transformer_manager.Register(std::move(entry), level); + } + }; - ORT_ENFORCE(graph_optimization_level <= TransformerLevel::MaxLevel, - "Exceeded max transformer level. Current level is set to " + - std::to_string(static_cast(graph_optimization_level))); + ORT_ENFORCE(graph_optimization_level <= TransformerLevel::MaxLevel, + "Exceeded max transformer level. Current level is set to " + + std::to_string(static_cast(graph_optimization_level))); - for (int i = static_cast(TransformerLevel::Level1); i <= static_cast(TransformerLevel::MaxLevel); i++) { - TransformerLevel level = static_cast(i); - if ((graph_optimization_level >= level) || !custom_list.empty()) { - add_transformers(level); - } + for (int i = static_cast(TransformerLevel::Level1); i <= static_cast(TransformerLevel::MaxLevel); i++) { + TransformerLevel level = static_cast(i); + if ((graph_optimization_level >= level) || !custom_list.empty()) { + add_transformers(level); } } +} #endif // !defined(ORT_MINIMAL_BUILD) - common::Status InferenceSession::WaitForNotification(Notification* p_executor_done, int64_t timeout_in_ms) { - if (timeout_in_ms > 0) { - ORT_NOT_IMPLEMENTED(__FUNCTION__, "timeout_in_ms >0 is not supported"); // TODO - } - p_executor_done->Wait(); - - return Status::OK(); +common::Status InferenceSession::WaitForNotification(Notification* p_executor_done, int64_t timeout_in_ms) { + if (timeout_in_ms > 0) { + ORT_NOT_IMPLEMENTED(__FUNCTION__, "timeout_in_ms >0 is not supported"); // TODO } + p_executor_done->Wait(); - SessionIOBinding::SessionIOBinding(InferenceSession* session) : sess_(session) { - ORT_ENFORCE(session->NewIOBinding(&binding_).IsOK()); - } + return Status::OK(); +} - InferenceSession* SessionIOBinding::GetInferenceSession() { - return sess_; - } +SessionIOBinding::SessionIOBinding(InferenceSession* session) : sess_(session) { + ORT_ENFORCE(session->NewIOBinding(&binding_).IsOK()); +} - IOBinding* SessionIOBinding::Get() { - return binding_.get(); - } +InferenceSession* SessionIOBinding::GetInferenceSession() { + return sess_; +} + +IOBinding* SessionIOBinding::Get() { + return binding_.get(); +} } // namespace onnxruntime diff --git a/onnxruntime/test/framework/ort_model_only_test.cc b/onnxruntime/test/framework/ort_model_only_test.cc index dcc0d10856..86c5229bd8 100644 --- a/onnxruntime/test/framework/ort_model_only_test.cc +++ b/onnxruntime/test/framework/ort_model_only_test.cc @@ -50,6 +50,7 @@ struct OrtModelTestInfo { std::vector output_names; std::function&)> output_verifier; std::vector> configs; + bool run_use_buffer{false}; }; static void RunOrtModel(const OrtModelTestInfo& test_info) { @@ -58,8 +59,21 @@ static void RunOrtModel(const OrtModelTestInfo& test_info) { for (const auto& config : test_info.configs) so.AddConfigEntry(config.first.c_str(), config.second.c_str()); + std::vector model_data; InferenceSessionGetGraphWrapper session_object{so, GetEnvironment()}; - ASSERT_STATUS_OK(session_object.Load(test_info.model_filename)); // infer type from filename + if (test_info.run_use_buffer) { + // Load the file into a buffer and use the buffer to create inference session + size_t num_bytes = 0; + ASSERT_STATUS_OK(Env::Default().GetFileLength(test_info.model_filename.c_str(), num_bytes)); + model_data.resize(num_bytes); + std::ifstream bytes_stream(test_info.model_filename, std::ifstream::in | std::ifstream::binary); + bytes_stream.read(model_data.data(), num_bytes); + bytes_stream.close(); + ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast(num_bytes))); + } else { + ASSERT_STATUS_OK(session_object.Load(test_info.model_filename)); // infer type from filename + } + ASSERT_STATUS_OK(session_object.Initialize()); std::vector fetches; @@ -311,8 +325,7 @@ TEST(OrtModelOnlyTests, SerializeToOrtFormatMLOps) { #endif // #if !defined(DISABLE_ML_OPS) #endif // #if !defined(ORT_MINIMAL_BUILD) -// test that we can deserialize and run a previously saved ORT format model -TEST(OrtModelOnlyTests, LoadOrtFormatModel) { +OrtModelTestInfo GetTestInfoForLoadOrtFormatModel() { OrtModelTestInfo test_info; test_info.model_filename = ORT_TSTR("testdata/ort_github_issue_4031.onnx.ort"); test_info.logid = "LoadOrtFormatModel"; @@ -330,13 +343,26 @@ TEST(OrtModelOnlyTests, LoadOrtFormatModel) { ASSERT_TRUE(output.Data()[0] == 125.f); }; + return test_info; +} + +// test that we can deserialize and run a previously saved ORT format model +TEST(OrtModelOnlyTests, LoadOrtFormatModel) { + OrtModelTestInfo test_info = GetTestInfoForLoadOrtFormatModel(); + RunOrtModel(test_info); +} + +// Load the model from a buffer instead of a file path +TEST(OrtModelOnlyTests, LoadOrtFormatModelFromBuffer) { + OrtModelTestInfo test_info = GetTestInfoForLoadOrtFormatModel(); + test_info.run_use_buffer = true; RunOrtModel(test_info); } #if !defined(DISABLE_ML_OPS) // test that we can deserialize and run a previously saved ORT format model // for a model with sequence and map outputs -TEST(OrtModelOnlyTests, LoadOrtFormatModelMLOps) { +OrtModelTestInfo GetTestInfoForLoadOrtFormatModelMLOps() { OrtModelTestInfo test_info; test_info.model_filename = ORT_TSTR("testdata/sklearn_bin_voting_classifier_soft.ort"); test_info.logid = "LoadOrtFormatModelMLOps"; @@ -370,8 +396,23 @@ TEST(OrtModelOnlyTests, LoadOrtFormatModelMLOps) { } }; + return test_info; +} + +// test that we can deserialize and run a previously saved ORT format model +// for a model with sequence and map outputs +TEST(OrtModelOnlyTests, LoadOrtFormatModelMLOps) { + OrtModelTestInfo test_info = GetTestInfoForLoadOrtFormatModelMLOps(); RunOrtModel(test_info); } + +// Load the model from a buffer instead of a file path +TEST(OrtModelOnlyTests, LoadOrtFormatModelMLOpsFromBuffer) { + OrtModelTestInfo test_info = GetTestInfoForLoadOrtFormatModelMLOps(); + test_info.run_use_buffer = true; + RunOrtModel(test_info); +} + #endif // !defined(DISABLE_ML_OPS) } // namespace test diff --git a/onnxruntime/test/testdata/ort_github_issue_4031.onnx.ort b/onnxruntime/test/testdata/ort_github_issue_4031.onnx.ort index 0fe711ee71..6d10929180 100644 Binary files a/onnxruntime/test/testdata/ort_github_issue_4031.onnx.ort and b/onnxruntime/test/testdata/ort_github_issue_4031.onnx.ort differ diff --git a/onnxruntime/test/testdata/sklearn_bin_voting_classifier_soft.ort b/onnxruntime/test/testdata/sklearn_bin_voting_classifier_soft.ort index 7eae709f43..eaef3784e7 100644 Binary files a/onnxruntime/test/testdata/sklearn_bin_voting_classifier_soft.ort and b/onnxruntime/test/testdata/sklearn_bin_voting_classifier_soft.ort differ