// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once #include "MapFeatureDescriptor.h" #include "SequenceFeatureDescriptor.h" #include "TensorFeatureDescriptor.h" #include "WinMLAdapter.h" namespace Windows::AI::MachineLearning { // SequenceBase // // This is the base class for all data based Sequence types. // // Supported derived classes: // Map, Map // template struct SequenceBase : public winrt::implements< SequenceBase, winml::ILearningModelFeatureValue, WinML::ISequenceFeatureValue, WinML::ILotusValueProviderPrivate> { using AbiMapStringToFloat = wfc::IMap; using AbiMapInt64BitToFloat = wfc::IMap; template struct ValidLotusType { using Type = T; }; template <> struct ValidLotusType { //using Type = std::map; using TKey = std::string; using TValue = float; using Type = std::pair, std::vector>; using ABIKey = winrt::hstring; using ABIValue = TValue; }; template <> struct ValidLotusType { //using Type = std::map; using TKey = int64_t; using TValue = float; using Type = std::pair, std::vector>; using ABIKey = TKey; using ABIValue = TValue; }; template void GetElementDescriptor(winml::ILearningModelFeatureDescriptor* result) { *result = TensorFeatureDescriptorFrom::CreateAnonymous( std::vector{1, 1, 1, 1}); } template <> void GetElementDescriptor>( winml::ILearningModelFeatureDescriptor* result) { // zero dimensional tensor has empty shape auto value_descriptor = WinML::TensorFeatureDescriptorFrom::CreateAnonymous( std::vector{}); *result = winrt::make( nullptr /* set to null as values are name-less */, nullptr /* set to null as values are description-less */, false /* set to false as values dont have required annotations */, winml::TensorKind::String /* key kind */, value_descriptor /* value kind */); } template <> void GetElementDescriptor>( winml::ILearningModelFeatureDescriptor* result) { // zero dimensional tensor has empty shape auto value_descriptor = WinML::TensorFeatureDescriptorFrom::CreateAnonymous( std::vector{}); *result = winrt::make( nullptr /* set to null as values are name-less */, nullptr /* set to null as values are description-less */, false /* set to false as values dont have required annotations */, winml::TensorKind::Int64 /* key kind */, value_descriptor /* value kind */); } using LotusSequence = std::vector::Type>; using ABISequence = wfc::IIterable; SequenceBase(const ABISequence& data) : data_(data) {} static winml::ILearningModelFeatureValue Create() { auto sequence = winrt::single_threaded_vector(); return winrt::make(sequence); } static winml::ILearningModelFeatureValue Create( const ABISequence& data) { return winrt::make(data); } // ILearningModelFeatureValue implementation winml::LearningModelFeatureKind Kind() { return winml::LearningModelFeatureKind::Sequence; } STDMETHOD(get_ElementDescriptor) ( winml::ILearningModelFeatureDescriptor* result) { FAIL_FAST_IF_NULL(result); GetElementDescriptor(result); return S_OK; } template static typename ValidLotusType::Type ConvertToValidLotusType( TRawType raw) { return raw; } template <> static typename ValidLotusType::Type ConvertToValidLotusType( winrt::hstring raw) { return WinML::Strings::UTF8FromHString(raw); } template <> static typename ValidLotusType::Type ConvertToValidLotusType( AbiMapStringToFloat raw) { std::vector::TKey> keys; std::vector::TValue> values; for (auto pair : raw) { auto key = WinML::Strings::UTF8FromHString(pair.Key()); keys.push_back(key); values.push_back(pair.Value()); } return std::make_pair(keys, values); } template <> static typename ValidLotusType::Type ConvertToValidLotusType( AbiMapInt64BitToFloat raw) { std::vector::TKey> keys; std::vector::TValue> values; for (const auto& pair : raw) { keys.push_back(pair.Key()); values.push_back(pair.Value()); } return std::make_pair(keys, values); } void ConvertToLotusSequence( const ABISequence& sequence) { LotusSequence lotus_sequence; std::transform( begin(sequence), end(sequence), std::back_inserter(lotus_sequence), [](const auto& value) { return ConvertToValidLotusType(value); }); lotus_data_ = std::make_unique(lotus_sequence); } template static Ort::Value CreateOrtMap(TLotusKey* keys, TLotusValue* values, size_t len) { // now create OrtValue wrappers over the buffers auto cpu_memory = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); std::vector shape = {static_cast(len)}; auto keys_ort_value = Ort::Value::CreateTensor(cpu_memory, keys, len, shape.data(), shape.size()); auto values_ort_value = Ort::Value::CreateTensor(cpu_memory, values, len, shape.data(), shape.size()); // make the map return Ort::Value::CreateMap(keys_ort_value, values_ort_value); } STDMETHOD(GetOrtValue)( WinML::BindingContext& context, OrtValue** ort_value) { // TODO: Tensorized data should be cached so multiple bindings work more efficiently // TODO : we need to handle inputs. for now only handle outputs and don't pre allocate anything if (context.type == WinML::BindingType::kOutput) { *ort_value = nullptr; return S_OK; } // handle inputs, create and store a copy of the sequence ConvertToLotusSequence(data_); // now create OrtValue wrappers over the buffers std::vector sequence_values; for (auto it = lotus_data_->begin(); it != lotus_data_->end(); ++it) { // make a ort value for this map auto map = *it; sequence_values.emplace_back(CreateOrtMap(map.first.data(), map.second.data(), map.first.size())); } *ort_value = Ort::Value::CreateSequence(sequence_values).release(); return S_OK; /* winrt::com_ptr adapter; RETURN_IF_FAILED(OrtGetWinMLAdapter(adapter.put())); auto lotus_type = adapter->GetVectorMapType( TensorKindFrom::TKey>::Type, TensorKindFrom::TValue>::Type); winrt::com_ptr ml_value_out; adapter->CreateOrtValue(lotus_data_.get(), lotus_type, ml_value_out.put()); *ml_value = ml_value_out.detach();*/ } STDMETHOD(IsPlaceholder) ( bool* p_is_placeholder) { FAIL_FAST_IF_NULL(p_is_placeholder); *p_is_placeholder = false; return S_OK; } template static std::vector ConvertToABIType(Ort::Value& ort_value) { // make sure this is an array of these types auto shape = ort_value.GetTensorTypeAndShapeInfo().GetShape(); // there needs to be only one dimension THROW_HR_IF(E_INVALIDARG, shape.size() != 1); auto lotus_value = ort_value.GetTensorMutableData::Type>(); // now go through all the entries std::vector out; for (auto i = 0; i < shape[0]; i++) { out.push_back(lotus_value[i]); } // return the vector return out; } template <> static std::vector ConvertToABIType(Ort::Value& ort_value) { auto strings = ort_value.GetStrings(); std::vector out; for (auto i = 0; i < strings.size(); ++i) { out.push_back(WinML::Strings::HStringFromUTF8(strings[i].c_str())); } return out; } STDMETHOD(UpdateSourceResourceData)( BindingContext& context, OrtValue* ort_value) { ORT_UNUSED_PARAMETER(context); auto writable_vector = data_.as>(); writable_vector.Clear(); Ort::AllocatorWithDefaultOptions allocator; size_t len; Ort::ThrowOnError(Ort::GetApi().GetValueCount(ort_value, &len)); for (auto i = 0; i < len; ++i) { OrtValue* out = nullptr; Ort::ThrowOnError(Ort::GetApi().GetValue(ort_value, i, allocator, &out)); Ort::Value map{out}; auto keys = map.GetValue(0, allocator); auto values = map.GetValue(1, allocator); auto keys_vector = ConvertToABIType::ABIKey>(keys); auto values_vector = ConvertToABIType::ABIValue>(values); std::map::ABIKey, typename ValidLotusType::ABIValue> std_map; for (auto j = 0; j < keys_vector.size(); ++j) { std_map[keys_vector[j]] = values_vector[j]; } auto abi_map = winrt::single_threaded_map::ABIKey, typename ValidLotusType::ABIValue>( std::move(std_map)); writable_vector.Append(abi_map); } return S_OK; } STDMETHOD(AbiRepresentation)( wf::IInspectable& abi_representation) { data_.as(abi_representation); return S_OK; } private: ABISequence data_; std::unique_ptr lotus_data_; }; } // namespace Windows::AI::MachineLearning