onnxruntime/winml/lib/Api/impl/SequenceBase.h

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#pragma once

#include "MapFeatureDescriptor.h"
#include "SequenceFeatureDescriptor.h"
#include "TensorFeatureDescriptor.h"

namespace _winml {

// SequenceBase
//
// This is the base class for all data based Sequence types.
//
// Supported derived classes:
//    Map<String, Float>, Map<Int64, Float>
//
template <typename TDerived, typename T, typename TRaw>
struct SequenceBase : public winrt::implements<
                          SequenceBase<TDerived, T, TRaw>,
                          winml::ILearningModelFeatureValue,
                          _winml::ISequenceFeatureValue,
                          _winml::ILotusValueProviderPrivate> {
  using ABISequence = wfc::IIterable<T>;
  using AbiMapStringToFloat = wfc::IMap<winrt::hstring, float>;
  using AbiMapInt64BitToFloat = wfc::IMap<int64_t, float>;

  static_assert(
      std::is_same<T, AbiMapStringToFloat>::value ||
          std::is_same<T, AbiMapInt64BitToFloat>::value ||
          std::is_same<TRaw, bool>::value ||
          std::is_same<TRaw, float>::value ||
          std::is_same<TRaw, double>::value ||
          std::is_same<TRaw, int8_t>::value ||
          std::is_same<TRaw, uint8_t>::value ||
          std::is_same<TRaw, uint16_t>::value ||
          std::is_same<TRaw, int16_t>::value ||
          std::is_same<TRaw, uint32_t>::value ||
          std::is_same<TRaw, int32_t>::value ||
          std::is_same<TRaw, uint64_t>::value ||
          std::is_same<TRaw, int64_t>::value ||
          std::is_same<TRaw, _winml::Half>::value ||
          std::is_same<TRaw, std::string>::value,
          "Only sequences of of map<string, float>, map<int64, float> and tensor<T> are supported.");

  template <typename T> struct SequenceAbiTypeInfo {
    static constexpr winml::TensorKind Key = winml::TensorKind::Undefined;
    static constexpr winml::TensorKind Value = winml::TensorKind::Undefined;
  };
  template <> struct SequenceAbiTypeInfo<AbiMapStringToFloat> {
    static constexpr winml::TensorKind Key = winml::TensorKind::String;
    static constexpr winml::TensorKind Value = winml::TensorKind::Float;
  };
  template <> struct SequenceAbiTypeInfo<AbiMapInt64BitToFloat> {
    static constexpr winml::TensorKind Key = winml::TensorKind::Int64;
    static constexpr winml::TensorKind Value = winml::TensorKind::Float;
  };

  template <typename TElement>
  void
  GetElementDescriptor(winml::ILearningModelFeatureDescriptor* result) {
    
    *result = _winml::TensorFeatureDescriptorFrom<TRaw>::CreateAnonymous(std::vector<int64_t>{});
  }

  template <>
  void
  GetElementDescriptor<wfc::IMap<winrt::hstring, float>>(
      winml::ILearningModelFeatureDescriptor* result) {
    // zero dimensional tensor has empty shape
    auto value_descriptor =
        _winml::TensorFeatureDescriptorFrom<float>::CreateAnonymous(
            std::vector<int64_t>{});
    *result =
        winrt::make<winmlp::MapFeatureDescriptor>(
            nullptr /* set to null as values are name-less */,
            nullptr /* set to null as values are description-less */,
            false /* set to false as values dont have required annotations */,
            winml::TensorKind::String /* key kind */,
            value_descriptor /* value kind */);
  }

  template <>
  void
  GetElementDescriptor<wfc::IMap<int64_t, float>>(
      winml::ILearningModelFeatureDescriptor* result) {
    // zero dimensional tensor has empty shape
    auto value_descriptor =
        _winml::TensorFeatureDescriptorFrom<float>::CreateAnonymous(
            std::vector<int64_t>{});
    *result =
        winrt::make<winmlp::MapFeatureDescriptor>(
            nullptr /* set to null as values are name-less */,
            nullptr /* set to null as values are description-less */,
            false /* set to false as values dont have required annotations */,
            winml::TensorKind::Int64 /* key kind */,
            value_descriptor /* value kind */);
  }

  SequenceBase(const ABISequence& data) : data_(data) {}

  static winml::ILearningModelFeatureValue
  Create() {
    auto sequence = winrt::single_threaded_vector<T>();
    return winrt::make<TDerived>(sequence);
  }

  static winml::ILearningModelFeatureValue
  Create(
      const ABISequence& data) {
    return winrt::make<TDerived>(data);
  }

  // ILearningModelFeatureValue implementation
  winml::LearningModelFeatureKind
  Kind() {
    return winml::LearningModelFeatureKind::Sequence;
  }

  STDMETHOD(get_ElementDescriptor)(
    winml::ILearningModelFeatureDescriptor* result) {
    FAIL_FAST_IF_NULL(result);

    GetElementDescriptor<T>(result);

    return S_OK;
  }

  STDMETHOD(GetValue)(
    _winml::BindingContext& context,
    IValue** out) {
    auto session = context.session.as<winmlp::LearningModelSession>();
    auto engine = session->GetEngine();

    if (context.type == _winml::BindingType::kInput) {
      winml::ILearningModelFeatureDescriptor descriptor(nullptr);
      GetElementDescriptor<T>(&descriptor);

      if (descriptor.Kind() == winml::LearningModelFeatureKind::Map) {
        // In opset 10 and earlier only seq<map<,>> were supported
        RETURN_IF_FAILED(engine->CreateSequenceOfMapsValue(
            reinterpret_cast<::IInspectable*>(winrt::get_abi(data_)),
            SequenceAbiTypeInfo<T>::Key, SequenceAbiTypeInfo<T>::Value, out));
      }
      else if (descriptor.Kind() == winml::LearningModelFeatureKind::Tensor) {
        // In opset 11, operators that require seq<tensor<t>> were added.

        // IVector<Tensor*> -> std::vector<IValue>
        //
        // Convert all of the data in the sequence of tensors IVector into the appropriate
        // IValues based on the session's EP. This is done by calling into each tensor's
        // GetValue and delegating tensorization to each of those objects.
        //
        // The resulting tensors are collected into a vector.
        std::vector<winrt::com_ptr<_winml::IValue>> sequence;
        for (auto tensor : data_) {
          auto value_provider = tensor.as<_winml::ILotusValueProviderPrivate>();
          winrt::com_ptr<_winml::IValue> out_value;
          RETURN_IF_FAILED(value_provider->GetValue(context, out_value.put()));
          sequence.push_back(out_value);
        }

        // The collection of IValues needs wrapped into a single IValue
        // which represents the sequence<tensor> value.
        std::vector<_winml::IValue*> sequence_values;
        std::transform(
            std::begin(sequence),
            std::end(sequence),
            std::back_inserter(sequence_values),
            [](auto value) {
              return value.get();
            });

        RETURN_IF_FAILED(engine->CreateSequenceOfValuesValue(sequence_values.data(), sequence_values.size(), out));
      } else {
        // This should never happen, as the static_assert at the beginning of the code should prevent this path
        // from even being hit.
        FAIL_FAST();
      }
    } else {
      RETURN_IF_FAILED(engine->CreateNullValue(out));
    }
    return S_OK;
  }

  STDMETHOD(IsPlaceholder)
  (
      bool* p_is_placeholder) {
    FAIL_FAST_IF_NULL(p_is_placeholder);
    *p_is_placeholder = false;
    return S_OK;
  }

  template <typename TElement = T> auto CreatePlaceholderTensor() { return TElement(nullptr); }
  template <> auto CreatePlaceholderTensor<winml::TensorBoolean>() { return winmlp::TensorBoolean::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorFloat>() { return winmlp::TensorFloat::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorDouble>() { return winmlp::TensorDouble::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorInt8Bit>() { return winmlp::TensorInt8Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorUInt8Bit>() { return winmlp::TensorUInt8Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorUInt16Bit>() { return winmlp::TensorUInt16Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorInt16Bit>() { return winmlp::TensorInt16Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorUInt32Bit>() { return winmlp::TensorUInt32Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorInt32Bit>() { return winmlp::TensorInt32Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorUInt64Bit>() { return winmlp::TensorUInt64Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorInt64Bit>() { return winmlp::TensorInt64Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorFloat16Bit>() { return winmlp::TensorFloat16Bit::Create(); }
  template <> auto CreatePlaceholderTensor<winml::TensorString>() { return winmlp::TensorString::Create(); }
  
  void AppendValue(
      _winml::BindingContext& context, wfc::IVector<T> data, winrt::com_ptr<_winml::IValue> value) {
    auto tensor = CreatePlaceholderTensor();
    auto value_provider = tensor.as<_winml::ILotusValueProviderPrivate>();
    WINML_THROW_IF_FAILED(value_provider->UpdateSourceResourceData(context, value.get()));
    data.Append(tensor);
  }  

  STDMETHOD(UpdateSourceResourceData)
  (BindingContext& context, IValue* out) {
    auto writable_vector = data_.as<wfc::IVector<T>>();
    writable_vector.Clear();

    auto session = context.session.as<winmlp::LearningModelSession>();
    auto engine = session->GetEngine();

    winml::ILearningModelFeatureDescriptor descriptor(nullptr);
    GetElementDescriptor<T>(&descriptor);

    if (descriptor.Kind() == winml::LearningModelFeatureKind::Map) {
      // In opset 10 and earlier only seq<map<,>> were supported
      RETURN_IF_FAILED(engine->FillSequenceOfMapsValue(reinterpret_cast<::IInspectable*>(winrt::get_abi(data_)), SequenceAbiTypeInfo<T>::Key, SequenceAbiTypeInfo<T>::Value, out));
    } else if (descriptor.Kind() == winml::LearningModelFeatureKind::Tensor) {
      // In opset 11, operators that require seq<tensor<t>> were added.
      std::vector<winrt::com_ptr<_winml::IValue>> tensor_values;
      RETURN_IF_FAILED(engine->GetSequenceOfTensorValues(out, tensor_values));

      for (auto tensor_value : tensor_values) {
        AppendValue(context, writable_vector, tensor_value);
      }
    } else {
      FAIL_FAST();
    }
    return S_OK;
  }

  STDMETHOD(AbiRepresentation)
  (
      wf::IInspectable& abi_representation) {
    data_.as(abi_representation);
    return S_OK;
  }

 private:
  ABISequence data_;
};

}  // namespace _winml