diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 6b0a3fa137..fd5f4a2aa4 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -18,7 +18,7 @@ set(WINML_TEST_INC_DIR function(set_winml_target_properties target) set_target_properties(${target} PROPERTIES - FOLDER "WinMLTest" + FOLDER "ONNXRuntimeTest/winml" CXX_STANDARD 17 CXX_STANDARD_REQUIRED YES CXX_EXTENSIONS NO diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 4489197c01..798459d3fc 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -273,6 +273,7 @@ struct Value : Base { size_t GetStringTensorDataLength() const; void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const; + std::vector GetStrings(); template T* GetTensorMutableData(); diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index c1098f16b1..26a764e84b 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -444,6 +444,33 @@ inline void Value::GetStringTensorContent(void* buffer, size_t buffer_length, si ThrowOnError(Global::api_.GetStringTensorContent(p_, buffer, buffer_length, offsets, offsets_count)); } +inline std::vector Value::GetStrings() { + std::vector out; + // make sure this is an array of strings + auto shape = this->GetTensorTypeAndShapeInfo().GetShape(); + // there needs to be only one dimension + if (shape.size() != 1) throw Ort::Exception("shape.size() != 1", ORT_INVALID_ARGUMENT); + // make a big buffer to hold all the string data + size_t buflen = this->GetStringTensorDataLength(); + std::vector buf(buflen); + std::vector offsets(shape[0]); + this->GetStringTensorContent(buf.data(), buf.size(), offsets.data(), offsets.size()); + // now go build all the strings + for (auto i = 0; i < shape[0]; ++i) { + std::string str; + size_t strlen = 0; + // are we on the last one? + if (i == (shape[0] - 1ll)) { + strlen = buflen - offsets[i]; + } else { + strlen = offsets[i + 1ll] - offsets[i]; + } + str.append(reinterpret_cast(buf.data() + offsets[i]), strlen); + out.push_back(str); + } + return out; +} + template T* Value::GetTensorMutableData() { T* out; diff --git a/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.cc b/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.cc index 3d25aa1fe3..9f63513e87 100644 --- a/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.cc +++ b/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.cc @@ -25,48 +25,17 @@ namespace GraphTransformerHelpers { - void RegisterGraphTransformers(onnxruntime::InferenceSession* lotusSession, bool registerLotusTransforms) + void RegisterGraphTransformers(onnxruntime::InferenceSession* lotusSession) { // Register Lotus graph transformers + // we were able to combine all of the winml/dml/ort work except for 2 transformers. + // these 2 are tracked by : + // Bug 22973884 : Fix issues with BatchNorm + Add and BatchNorm + Mul handling implicit inputs, and move from Winml to ORT // - // TODO: Work out issues controlling graph optimization passes through ORT's optimization level - // and rule list. In the meantime (and before new transformers are tested in Winml), passes - // are registered explicitly, and the optimization level is set to default above (no optimization). - // - // Issues: - // Why is UnsqueezeElimination not registered by name in ORT? - // Why are level 2 (default) transformers not run before partitioning, which the DML XP requires? - // Why are level2 transformers only enabled on the CPU provider in GenerateTransformers? - // Why does name filtering only apply to rule based graph transformers? - // Why is Matmul+Add not used when contrib ops are disabled? - - if (registerLotusTransforms) - { - lotusSession->RegisterGraphTransformer(std::move(std::make_unique()), onnxruntime::TransformerLevel::Level1); - } - std::unique_ptr rule_transformer = std::make_unique("WinmlRuleTransformer"); - - if (registerLotusTransforms) - { - rule_transformer->Register(std::make_unique()); - rule_transformer->Register(std::make_unique()); - rule_transformer->Register(std::make_unique()); - rule_transformer->Register(std::make_unique()); - rule_transformer->Register(std::make_unique()); - rule_transformer->Register(std::make_unique()); - rule_transformer->Register(std::make_unique()); - } - rule_transformer->Register(std::make_unique()); rule_transformer->Register(std::make_unique()); - lotusSession->RegisterGraphTransformer(std::move(rule_transformer), onnxruntime::TransformerLevel::Level1); - - if (registerLotusTransforms) - { - lotusSession->RegisterGraphTransformer(std::move(std::make_unique()), onnxruntime::TransformerLevel::Level1); - } } } \ No newline at end of file diff --git a/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.h b/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.h index 169597c0d1..bd9b1148cf 100644 --- a/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.h +++ b/onnxruntime/core/providers/dml/GraphTransformers/GraphTransformerHelpers.h @@ -5,5 +5,5 @@ namespace GraphTransformerHelpers { - void RegisterGraphTransformers(onnxruntime::InferenceSession* lotusSession, bool registerLotusTransforms); + void RegisterGraphTransformers(onnxruntime::InferenceSession* lotusSession); } \ No newline at end of file diff --git a/winml/lib/Api.Core/WinMLAdapter.cpp b/winml/lib/Api.Core/WinMLAdapter.cpp index e0ef547521..4dd12dad7a 100644 --- a/winml/lib/Api.Core/WinMLAdapter.cpp +++ b/winml/lib/Api.Core/WinMLAdapter.cpp @@ -574,12 +574,11 @@ HRESULT STDMETHODCALLTYPE GetOperatorRegistry(ILearningModelOperatorProviderNati return static_cast(this_)->impl_->Free(p); } static const struct OrtMemoryInfo* ORT_API_CALL InfoImpl(const struct OrtAllocator* this_) { - return &(static_cast(this_)->info_); + return &(static_cast(this_)->impl_->Info()); } private: onnxruntime::AllocatorPtr impl_; - OrtMemoryInfo info_; }; HRESULT STDMETHODCALLTYPE GetProviderAllocator( @@ -654,9 +653,10 @@ class IOBinding : public Microsoft::WRL::RuntimeClass< InferenceSession::InferenceSession(onnxruntime::InferenceSession* session) : session_(session) { } -void STDMETHODCALLTYPE InferenceSession::RegisterGraphTransformers(bool registerLotusTransforms) { +void STDMETHODCALLTYPE InferenceSession::RegisterGraphTransformers() { #ifdef USE_DML - GraphTransformerHelpers::RegisterGraphTransformers(session_.get(), registerLotusTransforms); + // Bug 22973884 : Fix issues with BatchNorm + Add and BatchNorm + Mul handling implicit inputs, and move from Winml to ORT + GraphTransformerHelpers::RegisterGraphTransformers(session_.get()); #endif USE_DML } diff --git a/winml/lib/Api.Core/inc/WinMLAdapter.h b/winml/lib/Api.Core/inc/WinMLAdapter.h index 38a835c52e..cb57534cac 100644 --- a/winml/lib/Api.Core/inc/WinMLAdapter.h +++ b/winml/lib/Api.Core/inc/WinMLAdapter.h @@ -39,7 +39,7 @@ MIDL_INTERFACE("a848faf6-5a2e-4a7f-b622-cc036f71e28a") IModelProto : IUnknown{ MIDL_INTERFACE("6ec766ef-6365-42bf-b64f-ae85c015adb8") IInferenceSession : IUnknown { virtual onnxruntime::InferenceSession* STDMETHODCALLTYPE get() = 0; - virtual void STDMETHODCALLTYPE RegisterGraphTransformers(bool registerLotusTransforms) = 0; + virtual void STDMETHODCALLTYPE RegisterGraphTransformers() = 0; virtual HRESULT STDMETHODCALLTYPE RegisterCustomRegistry(IMLOperatorRegistry * registry) = 0; virtual HRESULT STDMETHODCALLTYPE LoadModel(IModelProto* model_proto) = 0; virtual HRESULT STDMETHODCALLTYPE NewIOBinding(IIOBinding** io_binding) = 0; @@ -131,7 +131,7 @@ public: InferenceSession(onnxruntime::InferenceSession * session); onnxruntime::InferenceSession* STDMETHODCALLTYPE get() override { return session_.get(); } - void STDMETHODCALLTYPE RegisterGraphTransformers(bool registerLotusTransforms) override; + void STDMETHODCALLTYPE RegisterGraphTransformers() override; HRESULT STDMETHODCALLTYPE RegisterCustomRegistry(IMLOperatorRegistry* registry) override; HRESULT STDMETHODCALLTYPE LoadModel(IModelProto* model_proto) override; HRESULT STDMETHODCALLTYPE NewIOBinding(IIOBinding** io_binding) override; diff --git a/winml/lib/Api/LearningModelSession.cpp b/winml/lib/Api/LearningModelSession.cpp index 1b44f37709..9370fb8116 100644 --- a/winml/lib/Api/LearningModelSession.cpp +++ b/winml/lib/Api/LearningModelSession.cpp @@ -130,8 +130,7 @@ void LearningModelSession::Initialize() { WINML_THROW_IF_FAILED(session->RegisterCustomRegistry(model->GetOperatorRegistry())); // Register only the transformers not already in ORT - const bool registerLotusTransformers = false; - session->RegisterGraphTransformers(registerLotusTransformers); + session->RegisterGraphTransformers(); // Load the model into the session WINML_THROW_IF_FAILED(session->LoadModel(model_proto.get())); diff --git a/winml/lib/Api/impl/MapBase.h b/winml/lib/Api/impl/MapBase.h index 9d86f8d113..09357a716f 100644 --- a/winml/lib/Api/impl/MapBase.h +++ b/winml/lib/Api/impl/MapBase.h @@ -63,14 +63,29 @@ struct MapBase : winrt::implements< } template - static TRawType ConvertToABIType(const typename ValidLotusType::Type& lotusValue) { - TRawType copy = lotusValue; - return copy; + static std::vector ConvertToABIType(Ort::Value& ort_value) { + // make sure this is an array of these types + auto shape = ort_value.GetTensorTypeAndShapeInfo().GetShape(); + // there needs to be only one dimension + THROW_HR_IF(E_INVALIDARG, shape.size() != 1); + auto lotus_value = ort_value.GetTensorMutableData::Type>(); + // now go through all the entries + std::vector out; + for (auto i = 0; i < shape[0]; i++) { + out.push_back(lotus_value[i]); + } + // retun the vector + return out; } template <> - static typename winrt::hstring ConvertToABIType(const typename ValidLotusType::Type& lotusValue) { - return WinML::Strings::HStringFromUTF8(lotusValue); + static std::vector ConvertToABIType(Ort::Value& ort_value) { + auto strings = ort_value.GetStrings(); + std::vector out; + for (auto i = 0; i < strings.size(); ++i) { + out.push_back(WinML::Strings::HStringFromUTF8(strings[i].c_str())); + } + return out; } MapBase(ABIMap const& data) : data_(data) {} @@ -133,7 +148,7 @@ struct MapBase : winrt::implements< } template - static Ort::Value CreateOrtMap(TLotusKey * keys, TLotusValue * values, size_t len) { + static Ort::Value CreateOrtMap(TLotusKey* keys, TLotusValue* values, size_t len) { // now create OrtValue wrappers over the buffers auto cpu_memory = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); std::vector shape = {static_cast(len)}; @@ -149,6 +164,7 @@ struct MapBase : winrt::implements< // TODO : we need to handle inputs. for now only handle outputs and don't pre allocate anything if (context.type == WinML::BindingType::kOutput) { + *ort_value = nullptr; return S_OK; } @@ -156,7 +172,7 @@ struct MapBase : winrt::implements< ConvertToLotusMap(data_); // and make the map - *ort_value = CreateOrtMap(lotus_data_->first.data(), lotus_data_->second.data(), lotus_data_->first.size()); + *ort_value = CreateOrtMap(lotus_data_->first.data(), lotus_data_->second.data(), lotus_data_->first.size()).release(); return S_OK; } @@ -171,8 +187,25 @@ struct MapBase : winrt::implements< (BindingContext& context, OrtValue* ort_value) { data_.Clear(); - void* pResource = nullptr; - Ort::ThrowOnError(Ort::GetApi().GetTensorMutableData(ort_value, &pResource)); + Ort::AllocatorWithDefaultOptions allocator; + + // get the keys + OrtValue* ptr = nullptr; + Ort::ThrowOnError(Ort::GetApi().GetValue(ort_value, 0, allocator, &ptr)); + Ort::Value keys{ptr}; + // get the values + ptr = nullptr; + Ort::ThrowOnError(Ort::GetApi().GetValue(ort_value, 1, allocator, &ptr)); + Ort::Value values{ptr}; + + auto keys_vector = ConvertToABIType(keys); + auto values_vector = ConvertToABIType(values); + + auto len = keys.GetCount(); + for (auto i = 0; i < len; ++i) { + data_.Insert(keys_vector[i], values_vector[i]); + } + return S_OK; // TODO: code this //const LotusMap& map = *static_cast(pResource); diff --git a/winml/lib/Api/impl/SequenceBase.h b/winml/lib/Api/impl/SequenceBase.h index de6e49fc19..43ac2d3ed0 100644 --- a/winml/lib/Api/impl/SequenceBase.h +++ b/winml/lib/Api/impl/SequenceBase.h @@ -34,6 +34,8 @@ struct SequenceBase : public winrt::implements< using TKey = std::string; using TValue = float; using Type = std::pair, std::vector>; + using ABIKey = winrt::hstring; + using ABIValue = TValue; }; template <> struct ValidLotusType { @@ -41,6 +43,8 @@ struct SequenceBase : public winrt::implements< using TKey = int64_t; using TValue = float; using Type = std::pair, std::vector>; + using ABIKey = TKey; + using ABIValue = TValue; }; template @@ -189,8 +193,7 @@ struct SequenceBase : public winrt::implements< return Ort::Value::CreateMap(keys_ort_value, values_ort_value); } - STDMETHOD(GetOrtValue) - ( + STDMETHOD(GetOrtValue)( WinML::BindingContext& context, OrtValue** ort_value) { // TODO: Tensorized data should be cached so multiple bindings work more efficiently @@ -235,71 +238,59 @@ struct SequenceBase : public winrt::implements< } template - static TRawType - ConvertToABIType( - const typename ValidLotusType::Type& lotus_value) { - // make a copy - TRawType copy = lotus_value; - return copy; + static std::vector ConvertToABIType(Ort::Value& ort_value) { + // make sure this is an array of these types + auto shape = ort_value.GetTensorTypeAndShapeInfo().GetShape(); + // there needs to be only one dimension + THROW_HR_IF(E_INVALIDARG, shape.size() != 1); + auto lotus_value = ort_value.GetTensorMutableData::Type>(); + // now go through all the entries + std::vector out; + for (auto i = 0; i < shape[0]; i++) { + out.push_back(lotus_value[i]); + } + // return the vector + return out; } template <> - static winrt::hstring - ConvertToABIType( - const typename ValidLotusType::Type& lotus_value) { - return WinML::Strings::HStringFromUTF8(lotus_value); - } - - static AbiMapStringToFloat - ConvertToABIType( - typename ValidLotusType::TKey* keys, - typename ValidLotusType::TValue* values, - size_t len) { - // need to make a copy to convert std::string to hstring - std::map copy; - for (auto i = 0; i < len; ++i) { - auto key = WinML::Strings::HStringFromUTF8(keys[i]); - copy[key] = values[i]; + static std::vector ConvertToABIType(Ort::Value& ort_value) { + auto strings = ort_value.GetStrings(); + std::vector out; + for (auto i = 0; i < strings.size(); ++i) { + out.push_back(WinML::Strings::HStringFromUTF8(strings[i].c_str())); } - return winrt::single_threaded_map( - std::move(copy)); + return out; } - static AbiMapInt64BitToFloat - ConvertToABIType( - typename ValidLotusType::TKey* keys, - typename ValidLotusType::TValue* values, - size_t len) { - // need to make a copy since stl objects are not ABI safe. - std::map copy; - for (auto i = 0; i < len; ++i) { - copy[keys[i]] = values[i]; - } - return winrt::single_threaded_map( - std::move(copy)); - } - - STDMETHOD(UpdateSourceResourceData) - ( + STDMETHOD(UpdateSourceResourceData)( BindingContext& context, OrtValue* ort_value) { auto writable_vector = data_.as>(); writable_vector.Clear(); Ort::AllocatorWithDefaultOptions allocator; - Ort::Value ort_value_in(ort_value); - auto len = ort_value_in.GetCount(); + size_t len; + Ort::ThrowOnError(Ort::GetApi().GetValueCount(ort_value, &len)); for (auto i = 0; i < len; ++i) { - auto map = ort_value_in.GetValue(i, allocator); + OrtValue* out = nullptr; + Ort::ThrowOnError(Ort::GetApi().GetValue(ort_value, i, allocator, &out)); + Ort::Value map{out}; auto keys = map.GetValue(0, allocator); auto values = map.GetValue(1, allocator); - auto keys_data = keys.GetTensorMutableData::TKey>(); - auto values_data = keys.GetTensorMutableData::TValue>(); - - writable_vector.Append(ConvertToABIType(keys_data, values_data, keys.GetTensorTypeAndShapeInfo().GetElementCount())); - } + auto keys_vector = ConvertToABIType::ABIKey>(keys); + auto values_vector = ConvertToABIType::ABIValue>(values); + std::map::ABIKey, typename ValidLotusType::ABIValue> std_map; + for (auto j = 0; j < keys_vector.size(); ++j) { + std_map[keys_vector[j]] = values_vector[j]; + } + auto abi_map = winrt::single_threaded_map::ABIKey, typename ValidLotusType::ABIValue>( + std::move(std_map)); + + writable_vector.Append(abi_map); + } return S_OK; }