mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-16 21:00:14 +00:00
0b7048e7d6
Update winml to use #cores - #soc cores by Default as the number of intraopthreads --------- Co-authored-by: Sheil Kumar <sheilk@microsoft.com>
87 lines
3.7 KiB
C++
87 lines
3.7 KiB
C++
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
// Licensed under the MIT License.
|
|
|
|
#pragma once
|
|
|
|
#include "LearningModelSessionOptions.g.h"
|
|
#include <thread>
|
|
namespace WINMLP {
|
|
|
|
struct LearningModelSessionOptions : LearningModelSessionOptionsT<
|
|
LearningModelSessionOptions,
|
|
ILearningModelSessionOptionsNative,
|
|
ILearningModelSessionOptionsNative1> {
|
|
LearningModelSessionOptions();
|
|
|
|
LearningModelSessionOptions(const LearningModelSessionOptions& options);
|
|
|
|
uint32_t BatchSizeOverride();
|
|
void BatchSizeOverride(uint32_t value);
|
|
|
|
bool CloseModelOnSessionCreation();
|
|
void CloseModelOnSessionCreation(bool value);
|
|
|
|
wfc::IMapView<winrt::hstring, uint32_t> NamedDimensionOverrides();
|
|
void OverrideNamedDimension(winrt::hstring name, uint32_t value);
|
|
|
|
STDMETHOD(SetIntraOpNumThreadsOverride)
|
|
(uint32_t intraOpNumThreads);
|
|
|
|
uint32_t GetIntraOpNumThreads();
|
|
|
|
STDMETHOD(SetIntraOpThreadSpinning)
|
|
(boolean allowSpinning);
|
|
|
|
bool GetIntraOpThreadSpinning();
|
|
|
|
const gsl::span<const winrt::hstring> GetCustomOpLibraryPaths() noexcept;
|
|
|
|
void RegisterCustomOpsLibrary(const winrt::hstring& path) noexcept;
|
|
|
|
private:
|
|
// The batch size override property is used to inform the engine when the developer
|
|
// wants to explicitly set the batch size of a model to a fixed batch size.
|
|
//
|
|
// 0 : dont override the model batch definitions
|
|
// 1...n : override the model with the given batch size
|
|
//
|
|
// This value is a unsigned value, and users are not allowed to override models with a free batch size.
|
|
// If the model supports free dimensional batch sizes, the caller should provide 0, to not override.
|
|
//
|
|
// The default value here is 1 so that models with free dimension batch sizes (which is very common)
|
|
// can be optimized to fixed sizes.
|
|
uint32_t batch_size_override_ = 1;
|
|
|
|
// The close model on session creation property is used to inform the engine when the developer
|
|
// no longer needs the learningmodelsession after session creation.
|
|
// The engine can use the learning model during session creation to move resources rather than make copies.
|
|
//
|
|
// True : Move resources in the LearningModel in to the LearningModelSession
|
|
// False : Copy resources in the LearningModel to the LearningModelSession
|
|
//
|
|
// The default value here is False so that models are not automatically closed on session creation.
|
|
bool close_model_on_session_creation_ = false;
|
|
|
|
// Map of named input dimensions to concrete values.
|
|
// This informs the engine when the developer wants to explictily set a named dimension to a fixed value.
|
|
|
|
// 0 : the dimension present in the model should be honored.
|
|
// 1...n: override the named input dimension to the given value and optimize evaluations.
|
|
wfc::IMap<winrt::hstring, uint32_t> named_dim_overrides_ = winrt::single_threaded_map<winrt::hstring, uint32_t>();
|
|
|
|
// The intra operator num threads property is used to control the number of threads used in the threadpool for intra operator calculations.
|
|
// The default value here is the maximum number of logical cores to ensure that the default behavior of WinML always runs the fastest.
|
|
// WARNING: Setting a number higher than the maximum number of logical cores may result in an inefficient threadpool
|
|
uint32_t intra_op_num_threads_override_;
|
|
|
|
bool allow_thread_spinning_ = true;
|
|
|
|
std::vector<winrt::hstring> custom_ops_lib_paths_;
|
|
};
|
|
|
|
} // namespace WINMLP
|
|
|
|
namespace WINML::factory_implementation {
|
|
struct LearningModelSessionOptions
|
|
: LearningModelSessionOptionsT<LearningModelSessionOptions, implementation::LearningModelSessionOptions> {};
|
|
} // namespace WINML::factory_implementation
|