diff --git a/winml/api/Windows.AI.MachineLearning.idl b/winml/api/Windows.AI.MachineLearning.idl index 95eed26e3f..f60ec1adf1 100644 --- a/winml/api/Windows.AI.MachineLearning.idl +++ b/winml/api/Windows.AI.MachineLearning.idl @@ -33,7 +33,7 @@ import "windows.storage.idl"; namespace ROOT_NS.AI.MachineLearning { - [contractversion(3)] + [contractversion(4)] apicontract MachineLearningContract{}; //! Forward declarations @@ -213,6 +213,15 @@ namespace ROOT_NS.AI.MachineLearning //! BatchSizeOverride > 0 indicates the size of batch that will be used to override the model batch size and optimize evaluations. UInt32 BatchSizeOverride { get; set; }; + [contract(MachineLearningContract, 4)] + { + //! The OverrideNamedDimension method will allow the model compiler to use constant batch size performance optimizations when setting up the LearningModelSession. + //! The caller can specify the size of the dimension for a given named dimension. + //! dimension = 0 indicates that the dimension present in the model should be honored. + //! dimension > 0 indicates the size of the dimension that will be used to override the model "name" dimension and optimize evaluations. + void OverrideNamedDimension(String name, UInt32 dimension); + } + [contract(MachineLearningContract, 3)] { //! The CloseModelOnSessionCreation option will allow the LearningModelSession to take ownership of the LearningModel's diff --git a/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.cpp b/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.cpp index 30cd0d523e..964916002f 100644 --- a/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.cpp +++ b/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.cpp @@ -41,6 +41,12 @@ STDMETHODIMP OnnxruntimeEngineBuilder::CreateEngine(_winml::IEngine** out) { RETURN_HR_IF_NOT_OK_MSG(ort_api->AddFreeDimensionOverride(session_options.get(), DATA_BATCH, batch_size_override_.value()), ort_api); } + if (named_dimension_overrides_) { + for (const auto& override : named_dimension_overrides_) { + std::string narrow_name = std::wstring_convert>().to_bytes(override.Key().c_str()); + ort_api->AddFreeDimensionOverrideByName(session_options.get(), narrow_name.c_str(), override.Value()); + } + } RETURN_HR_IF_NOT_OK_MSG(ort_api->SetIntraOpNumThreads(session_options.get(), intra_op_num_threads_override_), ort_api); @@ -81,6 +87,12 @@ STDMETHODIMP OnnxruntimeEngineBuilder::SetBatchSizeOverride(uint32_t batch_size_ return S_OK; } + +STDMETHODIMP OnnxruntimeEngineBuilder::SetNamedDimensionOverrides(wfc::IMapView named_dimension_overrides) { + named_dimension_overrides_ = std::move(named_dimension_overrides); + return S_OK; +} + STDMETHODIMP OnnxruntimeEngineBuilder::SetIntraOpNumThreadsOverride(uint32_t intra_op_num_threads) { intra_op_num_threads_override_ = intra_op_num_threads; return S_OK; diff --git a/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.h b/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.h index bacdc44b92..d0cb7e2c94 100644 --- a/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.h +++ b/winml/lib/Api.Ort/OnnxruntimeEngineBuilder.h @@ -26,6 +26,9 @@ class OnnxruntimeEngineBuilder : public Microsoft::WRL::RuntimeClass< STDMETHOD(SetBatchSizeOverride) (uint32_t batch_size_override); + STDMETHOD(SetNamedDimensionOverrides) + (wfc::IMapView named_dimension_overrides); + STDMETHOD(SetIntraOpNumThreadsOverride) (uint32_t intra_op_num_threads); @@ -38,6 +41,7 @@ class OnnxruntimeEngineBuilder : public Microsoft::WRL::RuntimeClass< Microsoft::WRL::ComPtr queue_ = nullptr; bool metacommands_enabled_ = true; std::optional batch_size_override_; + wfc::IMapView named_dimension_overrides_; uint32_t intra_op_num_threads_override_; }; diff --git a/winml/lib/Api/LearningModelSession.cpp b/winml/lib/Api/LearningModelSession.cpp index 4697857969..4368f0a7a7 100644 --- a/winml/lib/Api/LearningModelSession.cpp +++ b/winml/lib/Api/LearningModelSession.cpp @@ -110,6 +110,7 @@ void LearningModelSession::Initialize() { WINML_THROW_IF_FAILED(engine_builder->SetMetacommandsEnabled(device_impl->MetacommandsEnabled())); } + // Make onnxruntime apply the batch size override, if any if (session_options_) { if (session_options_.BatchSizeOverride() != 0) { @@ -119,6 +120,12 @@ void LearningModelSession::Initialize() { uint32_t numIntraOpThreads = session_options_.as()->GetIntraOpNumThreads(); WINML_THROW_IF_FAILED(engine_builder->SetIntraOpNumThreadsOverride(numIntraOpThreads) ); + + // Make onnxruntime apply named dimension overrides, if any + com_ptr session_options_impl = session_options_.as(); + if (session_options_impl && session_options_impl->NamedDimensionOverrides().Size() > 0) { + WINML_THROW_IF_FAILED(engine_builder->SetNamedDimensionOverrides(session_options_impl->NamedDimensionOverrides())); + } } else { // Onnxruntime will use half the number of concurrent threads supported on the system // by default. This causes MLAS to not exercise every logical core. diff --git a/winml/lib/Api/LearningModelSessionOptions.cpp b/winml/lib/Api/LearningModelSessionOptions.cpp index 384ba7c15f..72c5c9a6ea 100644 --- a/winml/lib/Api/LearningModelSessionOptions.cpp +++ b/winml/lib/Api/LearningModelSessionOptions.cpp @@ -24,6 +24,14 @@ void LearningModelSessionOptions::CloseModelOnSessionCreation(bool value) { close_model_on_session_creation_ = value; } +wfc::IMapView LearningModelSessionOptions::NamedDimensionOverrides() { + return named_dim_overrides_.GetView(); +} + +void LearningModelSessionOptions::OverrideNamedDimension(winrt::hstring name, uint32_t value) { + named_dim_overrides_.Insert(name, value); +} + uint32_t LearningModelSessionOptions::GetIntraOpNumThreads() { return intra_op_num_threads_override_; } @@ -32,4 +40,4 @@ STDMETHODIMP LearningModelSessionOptions::SetIntraOpNumThreadsOverride(uint32_t intra_op_num_threads_override_ = intraOpNumThreads; return S_OK; } -} // namespace WINMLP \ No newline at end of file +} // namespace WINMLP diff --git a/winml/lib/Api/LearningModelSessionOptions.h b/winml/lib/Api/LearningModelSessionOptions.h index bbeb419e11..c4650dff1f 100644 --- a/winml/lib/Api/LearningModelSessionOptions.h +++ b/winml/lib/Api/LearningModelSessionOptions.h @@ -17,6 +17,9 @@ struct LearningModelSessionOptions : LearningModelSessionOptionsT NamedDimensionOverrides(); + void OverrideNamedDimension(winrt::hstring name, uint32_t value); STDMETHOD(SetIntraOpNumThreadsOverride) (uint32_t intraOpNumThreads); @@ -47,6 +50,13 @@ struct LearningModelSessionOptions : LearningModelSessionOptionsT named_dim_overrides_ = winrt::single_threaded_map(); + // The intra operator num threads property is used to control the number of threads used in the threadpool for intra operator calculations. // The default value here is the maximum number of logical cores to ensure that the default behavior of WinML always runs the fastest. // WARNING: Setting a number higher than the maximum number of logical cores may result in an inefficient threadpool diff --git a/winml/lib/Common/inc/iengine.h b/winml/lib/Common/inc/iengine.h index 0cfc08e384..37880095a0 100644 --- a/winml/lib/Common/inc/iengine.h +++ b/winml/lib/Common/inc/iengine.h @@ -168,6 +168,9 @@ IEngineBuilder : IUnknown { STDMETHOD(SetBatchSizeOverride) (uint32_t batch_size_override) PURE; + STDMETHOD(SetNamedDimensionOverrides) + (wfc::IMapView named_dimension_overrides) PURE; + STDMETHOD(SetIntraOpNumThreadsOverride) (uint32_t intra_op_num_threads) PURE; diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index 2163b8bf0b..b314fede2c 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -311,6 +311,40 @@ static void EvaluateSessionAndCloseModel() WINML_EXPECT_NO_THROW(::EvaluateSessionAndCloseModelHelper(LearningModelDeviceKind::Cpu, false)); } +static void NamedDimensionOverride() +{ + LearningModel model = nullptr; + WINML_EXPECT_NO_THROW(APITest::LoadModel(L"fns-candy.onnx", model)); + + LearningModelDevice device(nullptr); + WINML_EXPECT_NO_THROW(device = LearningModelDevice(LearningModelDeviceKind::Cpu)); + + // the model input shape. the batch size, n, is overriden to 5 + int64_t n = 5, c = 3, h = 720, w = 720; + + LearningModelSessionOptions options; + options.OverrideNamedDimension(L"None", n); + + // Verifies that if a Dim name doesn't exist the named dimension override does nothing + options.OverrideNamedDimension(L"DimNameThatDoesntExist", n); + + LearningModelSession session(nullptr); + WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device, options)); + + ILearningModelFeatureDescriptor descriptor = model.InputFeatures().GetAt(0); + TensorFeatureDescriptor tensorDescriptor = nullptr; + descriptor.as(tensorDescriptor); + std::vector shape{n,c,h,w}; + int64_t size = n*c*h*w; + std::vector buffer; + buffer.resize(static_cast(size)); + auto featureValue = TensorFloat::CreateFromIterable(shape, winrt::single_threaded_vector(std::move(buffer))); + LearningModelBinding binding(session); + binding.Bind(descriptor.Name(), featureValue); + + WINML_EXPECT_NO_THROW(session.Evaluate(binding, L"")); +} + static void CloseSession() { LearningModel learningModel = nullptr; @@ -435,6 +469,7 @@ const LearningModelSessionAPITestsApi& getapi() { CreateSessionWithCastToFloat16InModel, CreateSessionWithFloat16InitializersInModel, EvaluateSessionAndCloseModel, + NamedDimensionOverride, CloseSession, SetIntraOpNumThreads }; diff --git a/winml/test/api/LearningModelSessionAPITest.h b/winml/test/api/LearningModelSessionAPITest.h index b544120d2d..da630527b9 100644 --- a/winml/test/api/LearningModelSessionAPITest.h +++ b/winml/test/api/LearningModelSessionAPITest.h @@ -18,6 +18,7 @@ struct LearningModelSessionAPITestsApi { VoidTest CreateSessionWithCastToFloat16InModel; VoidTest CreateSessionWithFloat16InitializersInModel; VoidTest EvaluateSessionAndCloseModel; + VoidTest OverrideNamedDimension; VoidTest CloseSession; VoidTest SetIntraOpNumThreads; }; @@ -39,5 +40,7 @@ WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectXMinimumPower) WINML_TEST(LearningModelSessionAPITests, CreateSessionWithCastToFloat16InModel) WINML_TEST(LearningModelSessionAPITests, CreateSessionWithFloat16InitializersInModel) WINML_TEST(LearningModelSessionAPITests, AdapterIdAndDevice) +WINML_TEST(LearningModelSessionAPITests, OverrideNamedDimension) +WINML_TEST(LearningModelSessionAPITests, CloseSession) WINML_TEST(LearningModelSessionAPITests, SetIntraOpNumThreads) WINML_TEST_CLASS_END()