onnxruntime/winml/test/api/LearningModelBindingAPITest.cpp

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "testPch.h"

#include "APITest.h"
#include "LearningModelBindingAPITest.h"
#include "SqueezeNetValidator.h"

#include <sstream>

using namespace winrt;
using namespace winml;
using namespace wfc;
using namespace wgi;
using namespace wm;
using namespace ws;

static void LearningModelBindingAPITestsClassSetup() {
  init_apartment();
#ifdef BUILD_INBOX
  winrt_activation_handler = WINRT_RoGetActivationFactory;
#endif
}

static void CpuSqueezeNet() {
  std::string cpuInstance("CPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
    cpuInstance,
    LearningModelDeviceKind::Cpu,
    /*dataTolerance*/ 0.00001f,
    false
  ));
}

static void CpuSqueezeNetEmptyOutputs() {
  std::string cpuInstance("CPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
                          cpuInstance,
                          LearningModelDeviceKind::Cpu,
                          /*dataTolerance*/ 0.00001f,
                          false,
                          OutputBindingStrategy::Empty
  ););
}

static void CpuSqueezeNetUnboundOutputs() {
  std::string cpuInstance("CPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
                          cpuInstance,
                          LearningModelDeviceKind::Cpu,
                          /*dataTolerance*/ 0.00001f,
                          false,
                          OutputBindingStrategy::Unbound
  ););
}

static void CpuSqueezeNetBindInputTensorAsInspectable() {
  std::string cpuInstance("CPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
                          cpuInstance,
                          LearningModelDeviceKind::Cpu,
                          /*dataTolerance*/ 0.00001f,
                          false,
                          OutputBindingStrategy::Bound /* empty outputs */,
                          true /* bind inputs as inspectables */
  ););
}

static void CastMapInt64() {
  WINML_EXPECT_NO_THROW(LearningModel::LoadFromFilePath(FileHelpers::GetModulePath() + L"castmap-int64.onnx"));
  // TODO: Check Descriptor
}

static void DictionaryVectorizerMapInt64() {
  LearningModel learningModel = nullptr;
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-int64.onnx", learningModel));

  auto inputDescriptor = learningModel.InputFeatures().First().Current();
  WINML_EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map);
  auto mapDescriptor = inputDescriptor.as<MapFeatureDescriptor>();
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64);
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
  // empty size means tensor of scalar value
  WINML_EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0);
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);

  LearningModelSession modelSession(learningModel);
  LearningModelBinding binding(modelSession);
  std::unordered_map<int64_t, float> map;
  map[1] = 1.f;
  map[10] = 10.f;
  map[3] = 3.f;

  auto mapInputName = inputDescriptor.Name();

  // Bind as IMap
  auto abiMap = winrt::single_threaded_map(std::move(map));
  binding.Bind(mapInputName, abiMap);
  auto mapInputInspectable = abiMap.as<wf::IInspectable>();
  auto first = binding.First();
  WINML_EXPECT_TRUE(first.Current().Key() == mapInputName);
  WINML_EXPECT_TRUE(first.Current().Value() == mapInputInspectable);
  WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable);

  // Bind as IMapView
  auto mapView = abiMap.GetView();
  binding.Bind(mapInputName, mapView);
  mapInputInspectable = mapView.as<wf::IInspectable>();
  first = binding.First();
  WINML_EXPECT_TRUE(first.Current().Key() == mapInputName);
  WINML_EXPECT_TRUE(first.Current().Value() == mapView);
  WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapView);
}

static void DictionaryVectorizerMapString() {
  LearningModel learningModel = nullptr;
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-string.onnx", learningModel));

  auto inputDescriptor = learningModel.InputFeatures().First().Current();
  WINML_EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map);

  auto mapDescriptor = inputDescriptor.as<MapFeatureDescriptor>();
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String);
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);

  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
  // empty size means tensor of scalar value
  WINML_EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0);
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);

  LearningModelSession modelSession(learningModel);
  LearningModelBinding binding(modelSession);
  std::unordered_map<winrt::hstring, float> map;
  map[L"1"] = 1.f;
  map[L"10"] = 10.f;
  map[L"2"] = 2.f;

  auto mapInputName = inputDescriptor.Name();
  auto abiMap = winrt::single_threaded_map(std::move(map));
  binding.Bind(mapInputName, abiMap);

  auto mapInputInspectable = abiMap.as<wf::IInspectable>();
  auto first = binding.First();
  WINML_EXPECT_TRUE(first.Current().Key() == mapInputName);
  WINML_EXPECT_TRUE(first.Current().Value() == mapInputInspectable);
  WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable);

  modelSession.Evaluate(binding, L"");
}

static void RunZipMapInt64(winml::LearningModel model, OutputBindingStrategy bindingStrategy) {
  auto outputFeatures = model.OutputFeatures();
  auto outputDescriptor = outputFeatures.First().Current();
  WINML_EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence);

  auto seqDescriptor = outputDescriptor.as<SequenceFeatureDescriptor>();
  auto mapDescriptor = seqDescriptor.ElementDescriptor().as<MapFeatureDescriptor>();
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64);

  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);

  LearningModelSession session(model);
  LearningModelBinding binding(session);

  std::vector<float> inputs = {0.5f, 0.25f, 0.125f};
  std::vector<int64_t> shape = {1, 3};

  // Bind inputs
  auto inputTensor = TensorFloat::CreateFromArray(shape, winrt::array_view<const float>(std::move(inputs)));
  binding.Bind(winrt::hstring(L"X"), inputTensor);

  typedef IMap<int64_t, float> ABIMap;
  typedef IVector<ABIMap> ABISequeneceOfMap;

  ABISequeneceOfMap abiOutput = nullptr;
  // Bind outputs
  if (bindingStrategy == OutputBindingStrategy::Bound) {
    abiOutput = winrt::single_threaded_vector<ABIMap>();
    binding.Bind(winrt::hstring(L"Y"), abiOutput);
  }

  // Evaluate
  auto result = session.Evaluate(binding, L"0").Outputs();

  if (bindingStrategy == OutputBindingStrategy::Bound) {
    // from output binding
    const auto& out1 = abiOutput.GetAt(0);
    const auto& out2 = result.Lookup(L"Y").as<IVectorView<ABIMap>>().GetAt(0);
    WINML_LOG_COMMENT((std::ostringstream() << "size: " << out1.Size()).str());
    // check outputs
    auto iter1 = out1.First();
    auto iter2 = out2.First();
    for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) {
      WINML_EXPECT_TRUE(iter1.HasCurrent());
      WINML_EXPECT_TRUE(iter2.HasCurrent());
      const auto& pair1 = iter1.Current();
      const auto& pair2 = iter2.Current();
      WINML_LOG_COMMENT((std::ostringstream() << "key: " << pair1.Key() << ", value: " << pair2.Value()).str());
      WINML_EXPECT_TRUE(pair1.Key() == i && pair2.Key() == i);
      WINML_EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]);
      iter1.MoveNext();
      iter2.MoveNext();
    }
    WINML_EXPECT_TRUE(!iter1.HasCurrent());
    WINML_EXPECT_TRUE(!iter2.HasCurrent());
  } else {
    abiOutput = result.Lookup(L"Y").as<ABISequeneceOfMap>();
    WINML_EXPECT_TRUE(abiOutput.Size() == 1);
    ABIMap map = abiOutput.GetAt(0);
    WINML_EXPECT_TRUE(map.Size() == 3);
    WINML_EXPECT_TRUE(map.Lookup(0) == 0.5);
    WINML_EXPECT_TRUE(map.Lookup(1) == .25);
    WINML_EXPECT_TRUE(map.Lookup(2) == .125);
  }
}

static void ZipMapInt64() {
  LearningModel learningModel = nullptr;
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel));
  RunZipMapInt64(learningModel, OutputBindingStrategy::Bound);
}

static void ZipMapInt64Unbound() {
  LearningModel learningModel = nullptr;
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel));
  RunZipMapInt64(learningModel, OutputBindingStrategy::Unbound);
}

static void ZipMapString() {
  // output constraint: "seq(map(string, float))" or "seq(map(int64, float))"
  LearningModel learningModel = nullptr;
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-string.onnx", learningModel));
  auto outputs = learningModel.OutputFeatures();
  auto outputDescriptor = outputs.First().Current();
  WINML_EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence);
  auto mapDescriptor = outputDescriptor.as<SequenceFeatureDescriptor>().ElementDescriptor().as<MapFeatureDescriptor>();
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String);
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);

  LearningModelSession session(learningModel);
  LearningModelBinding binding(session);

  std::vector<float> inputs = {0.5f, 0.25f, 0.125f};
  std::vector<int64_t> shape = {1, 3};
  std::vector<winrt::hstring> labels = {L"cat", L"dog", L"lion"};
  std::map<winrt::hstring, float> mapData = {
    { L"cat", 0.0f},
    { L"dog", 0.0f},
    {L"lion", 0.0f}
  };
  typedef IMap<winrt::hstring, float> ABIMap;
  ABIMap abiMap = winrt::single_threaded_map<winrt::hstring, float>(std::move(mapData));
  std::vector<ABIMap> seqOutput = {abiMap};
  IVector<ABIMap> ABIOutput = winrt::single_threaded_vector<ABIMap>(std::move(seqOutput));

  TensorFloat inputTensor = TensorFloat::CreateFromArray(shape, winrt::array_view<const float>(std::move(inputs)));
  binding.Bind(winrt::hstring(L"X"), inputTensor);
  binding.Bind(winrt::hstring(L"Y"), ABIOutput);
  auto result = session.Evaluate(binding, L"0").Outputs();
  // from output binding
  const auto& out1 = ABIOutput.GetAt(0);
  const auto& out2 = result.Lookup(L"Y").as<IVectorView<ABIMap>>().GetAt(0);
  WINML_LOG_COMMENT((std::ostringstream() << "size: " << out1.Size()).str());
  // single key,value pair for each map
  auto iter1 = out1.First();
  auto iter2 = out2.First();
  for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) {
    WINML_EXPECT_TRUE(iter2.HasCurrent());
    const auto& pair1 = iter1.Current();
    const auto& pair2 = iter2.Current();
    WINML_LOG_COMMENT((std::ostringstream() << "key: " << pair1.Key().c_str() << ", value " << pair2.Value()).str());
    WINML_EXPECT_TRUE(std::wstring(pair1.Key().c_str()).compare(labels[i]) == 0);
    WINML_EXPECT_TRUE(std::wstring(pair2.Key().c_str()).compare(labels[i]) == 0);
    WINML_EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]);
    iter1.MoveNext();
    iter2.MoveNext();
  }
  WINML_EXPECT_TRUE(!iter1.HasCurrent());
  WINML_EXPECT_TRUE(!iter2.HasCurrent());
}

static void GpuSqueezeNet() {
  std::string gpuInstance("GPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
                          gpuInstance,
                          LearningModelDeviceKind::DirectX,
                          /*dataTolerance*/ 0.00001f
  ););
}

static void GpuSqueezeNetEmptyOutputs() {
  std::string gpuInstance("GPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
                          gpuInstance,
                          LearningModelDeviceKind::DirectX,
                          /*dataTolerance*/ 0.00001f,
                          false,
                          OutputBindingStrategy::Empty
  ););
}

static void GpuSqueezeNetUnboundOutputs() {
  std::string gpuInstance("GPU");
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
                          gpuInstance,
                          LearningModelDeviceKind::DirectX,
                          /*dataTolerance*/ 0.00001f,
                          false,
                          OutputBindingStrategy::Unbound
  ););
}

// Validates that when the input image is the same as the model expects, the binding step is executed correctly.
static void ImageBindingDimensions() {
  LearningModelBinding learningModelBinding = nullptr;
  LearningModel learningModel = nullptr;
  LearningModelSession learningModelSession = nullptr;
  LearningModelDevice leraningModelDevice = nullptr;
  std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx";
  // load a model with expected input size: 224 x 224
  WINML_EXPECT_NO_THROW(leraningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
  WINML_EXPECT_TRUE(learningModel != nullptr);
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, leraningModelDevice));
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));

  // Create input images and execute bind
  // Test Case 1: both width and height are larger than model expects
  VideoFrame inputImage1(BitmapPixelFormat::Rgba8, 1000, 1000);
  ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage1);
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));

  // Test Case 2: only height is larger, while width is smaller
  VideoFrame inputImage2(BitmapPixelFormat::Rgba8, 20, 1000);
  inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage2);
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));

  // Test Case 3: only width is larger, while height is smaller
  VideoFrame inputImage3(BitmapPixelFormat::Rgba8, 1000, 20);
  inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage3);
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));

  // Test Case 4: both width and height are smaller than model expects
  VideoFrame inputImage4(BitmapPixelFormat::Rgba8, 20, 20);
  inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage4);
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));
}

static void VerifyInvalidBindExceptions() {
  LearningModel learningModel = nullptr;
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel));

  LearningModelSession session(learningModel);
  LearningModelBinding binding(session);

  std::vector<float> inputs = {0.5f, 0.25f, 0.125f};
  std::vector<int64_t> shape = {1, 3};

  auto matchException = [](const winrt::hresult_error& e, HRESULT hr) -> bool { return e.code() == hr; };

  auto ensureWinmlSizeMismatch = std::bind(matchException, std::placeholders::_1, WINML_ERR_SIZE_MISMATCH);
  auto ensureWinmlInvalidBinding = std::bind(matchException, std::placeholders::_1, WINML_ERR_INVALID_BINDING);

  /*
    Verify tensor bindings throw correct bind exceptions
  */

  // Bind invalid image as tensorfloat input
  auto image = FileHelpers::LoadImageFeatureValue(L"227x227.png");
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", image), winrt::hresult_error, ensureWinmlSizeMismatch);

  // Bind invalid map as tensorfloat input
  std::unordered_map<float, float> map;
  auto abiMap = winrt::single_threaded_map(std::move(map));
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid sequence as tensorfloat input
  std::vector<uint32_t> sequence;
  auto abiSequence = winrt::single_threaded_vector(std::move(sequence));
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid tensor size as tensorfloat input
  auto tensorBoolean = TensorBoolean::Create();
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid tensor shape as tensorfloat input
  auto tensorInvalidShape = TensorFloat::Create(std::vector<int64_t>{2, 3, 4});
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding);

  /*
    Verify sequence bindings throw correct bind exceptions
  */

  // Bind invalid image as sequence<map<int, float> output
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", image), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid map as sequence<map<int, float> output
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid sequence<int> as sequence<map<int, float> output
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid tensor as sequence<map<int, float> output
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding);

  /*
    Verify map bindings throw correct bind exceptions
  */
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-int64.onnx", learningModel));

  LearningModelSession mapSession(learningModel);
  LearningModelBinding mapBinding(mapSession);

  auto inputName = learningModel.InputFeatures().First().Current().Name();

  // Bind invalid image as image input
  auto smallImage = FileHelpers::LoadImageFeatureValue(L"100x100.png");
  WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, smallImage), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid map as image input
  WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid sequence as image input
  WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding);

  // Bind invalid tensor type as image input
  WINML_EXPECT_THROW_SPECIFIC(
    mapBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding
  );
}

// Verify that it throws an error when binding an invalid name.
static void BindInvalidInputName() {
  LearningModel learningModel = nullptr;
  LearningModelBinding learningModelBinding = nullptr;
  LearningModelDevice learningModelDevice = nullptr;
  LearningModelSession learningModelSession = nullptr;
  std::wstring modelPath = FileHelpers::GetModulePath() + L"Add_ImageNet1920.onnx";
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(modelPath));
  WINML_EXPECT_TRUE(learningModel != nullptr);
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));

  VideoFrame iuputImage(BitmapPixelFormat::Rgba8, 1920, 1080);
  ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(iuputImage);

  auto first = learningModel.InputFeatures().First();
  std::wstring testInvalidName = L"0";

  // Verify that testInvalidName is not in model's InputFeatures
  while (first.HasCurrent()) {
    WINML_EXPECT_NOT_EQUAL(testInvalidName, first.Current().Name());
    first.MoveNext();
  }

  // Bind inputTensor to a valid input name
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"input_39:0", inputTensor));

  // Bind inputTensor to an invalid input name
  WINML_EXPECT_THROW_SPECIFIC(
    learningModelBinding.Bind(testInvalidName, inputTensor),
    winrt::hresult_error,
    [](const winrt::hresult_error& e) -> bool { return e.code() == WINML_ERR_INVALID_BINDING; }
  );
}

static void VerifyOutputAfterEvaluateAsyncCalledTwice() {
  LearningModel learningModel = nullptr;
  LearningModelBinding learningModelBinding = nullptr;
  LearningModelDevice learningModelDevice = nullptr;
  LearningModelSession learningModelSession = nullptr;
  std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx";
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
  WINML_EXPECT_TRUE(learningModel != nullptr);
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));

  auto inputShape = std::vector<int64_t>{5};
  auto inputData1 = std::vector<float>{-50.f, -25.f, 0.f, 25.f, 50.f};
  auto inputValue1 =
    TensorFloat::CreateFromIterable(inputShape, single_threaded_vector<float>(std::move(inputData1)).GetView());

  auto inputData2 = std::vector<float>{50.f, 25.f, 0.f, -25.f, -50.f};
  auto inputValue2 =
    TensorFloat::CreateFromIterable(inputShape, single_threaded_vector<float>(std::move(inputData2)).GetView());

  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", inputValue1));

  auto outputValue = TensorFloat::Create();
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"Y", outputValue));

  WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L""));

  auto buffer1 = outputValue.GetAsVectorView();
  WINML_EXPECT_TRUE(buffer1 != nullptr);

  // The second evaluation
  // If we don't bind output again, the output value will not change
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", inputValue2));
  WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L""));
  auto buffer2 = outputValue.GetAsVectorView();
  WINML_EXPECT_EQUAL(buffer1.Size(), buffer2.Size());
  bool isSame = true;
  for (uint32_t i = 0; i < buffer1.Size(); ++i) {
    if (buffer1.GetAt(i) != buffer2.GetAt(i)) {
      isSame = false;
      break;
    }
  }
  WINML_EXPECT_FALSE(isSame);
}

static VideoFrame CreateVideoFrame(const wchar_t* path) {
  auto imagefile = StorageFile::GetFileFromPathAsync(path).get();
  auto stream = imagefile.OpenAsync(FileAccessMode::Read).get();
  auto decoder = BitmapDecoder::CreateAsync(stream).get();
  auto softwareBitmap = decoder.GetSoftwareBitmapAsync().get();
  return VideoFrame::CreateWithSoftwareBitmap(softwareBitmap);
}

static void VerifyOutputAfterImageBindCalledTwice() {
  std::wstring fullModelPath = FileHelpers::GetModulePath() + L"model.onnx";
  std::wstring fullImagePath1 = FileHelpers::GetModulePath() + L"kitten_224.png";
  std::wstring fullImagePath2 = FileHelpers::GetModulePath() + L"fish.png";

  // winml model creation
  LearningModel model = nullptr;
  WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath));
  LearningModelSession modelSession = nullptr;
  WINML_EXPECT_NO_THROW(
    modelSession = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::Default))
  );
  LearningModelBinding modelBinding(modelSession);

  // create the tensor for the actual output
  auto output = TensorFloat::Create();
  modelBinding.Bind(L"softmaxout_1", output);

  // Bind image 1 and evaluate
  auto frame = CreateVideoFrame(fullImagePath1.c_str());
  auto imageTensor = ImageFeatureValue::CreateFromVideoFrame(frame);
  WINML_EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor));
  WINML_EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L""));

  // Store 1st result
  auto outputVectorView1 = output.GetAsVectorView();

  // Bind image 2 and evaluate
  // In this scenario, the backing videoframe is updated, and the imagefeaturevalue is rebound.
  // The expected result is that the videoframe will be re-tensorized at bind
  auto frame2 = CreateVideoFrame(fullImagePath2.c_str());
  frame2.CopyToAsync(frame).get();
  WINML_EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor));
  WINML_EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L""));

  // Store 2nd result
  auto outputVectorView2 = output.GetAsVectorView();

  WINML_EXPECT_EQUAL(outputVectorView1.Size(), outputVectorView2.Size());
  bool isSame = true;
  for (uint32_t i = 0; i < outputVectorView1.Size(); ++i) {
    if (outputVectorView1.GetAt(i) != outputVectorView2.GetAt(i)) {
      isSame = false;
      break;
    }
  }
  WINML_EXPECT_FALSE(isSame);
}

static void SequenceLengthTensorFloat() {
  // Tests sequence of tensor float as an input
  LearningModel learningModel = nullptr;
  LearningModelBinding learningModelBinding = nullptr;
  LearningModelDevice learningModelDevice = nullptr;
  LearningModelSession learningModelSession = nullptr;
  std::wstring filePath = FileHelpers::GetModulePath() + L"sequence_length.onnx";
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
  WINML_EXPECT_TRUE(learningModel != nullptr);
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));

  auto input = winrt::single_threaded_vector<TensorFloat>();
  for (int i = 0; i < 3; i++) {
    std::vector<int64_t> shape = {5, 3 * i + 1};
    std::vector<float> data(
      static_cast<size_t>(std::accumulate(shape.begin(), shape.end(), static_cast<int64_t>(1), std::multiplies()))
    );
    input.Append(TensorFloat::CreateFromShapeArrayAndDataArray(shape, data));
  }

  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", input));
  auto results = learningModelSession.Evaluate(learningModelBinding, L"");

  WINML_EXPECT_EQUAL(3, results.Outputs().Lookup(L"Y").as<TensorInt64Bit>().GetAsVectorView().GetAt(0));
}

static void SequenceConstructTensorString() {
  LearningModel learningModel = nullptr;
  LearningModelBinding learningModelBinding = nullptr;
  LearningModelDevice learningModelDevice = nullptr;
  LearningModelSession learningModelSession = nullptr;
  std::wstring filePath = FileHelpers::GetModulePath() + L"sequence_construct.onnx";
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
  WINML_EXPECT_TRUE(learningModel != nullptr);
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));

  std::vector<int64_t> shape1 = {2, 3};
  std::vector<int64_t> data1(
    static_cast<size_t>(std::accumulate(shape1.begin(), shape1.end(), static_cast<int64_t>(1), std::multiplies()))
  );
  auto input1 = TensorInt64Bit::CreateFromShapeArrayAndDataArray(shape1, data1);
  std::vector<int64_t> shape2 = {2, 3};
  std::vector<int64_t> data2(
    static_cast<size_t>(std::accumulate(shape2.begin(), shape2.end(), static_cast<int64_t>(1), std::multiplies()))
  );
  auto input2 = TensorInt64Bit::CreateFromShapeArrayAndDataArray(shape2, data2);

  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"tensor1", input1));
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"tensor2", input2));
  auto results = learningModelSession.Evaluate(learningModelBinding, L"");

  auto output_sequence = results.Outputs().Lookup(L"output_sequence").as<wfc::IVectorView<TensorInt64Bit>>();
  WINML_EXPECT_EQUAL(static_cast<uint32_t>(2), output_sequence.Size());
  WINML_EXPECT_EQUAL(2, output_sequence.GetAt(0).Shape().GetAt(0));
  WINML_EXPECT_EQUAL(3, output_sequence.GetAt(0).Shape().GetAt(1));
  WINML_EXPECT_EQUAL(2, output_sequence.GetAt(1).Shape().GetAt(0));
  WINML_EXPECT_EQUAL(3, output_sequence.GetAt(1).Shape().GetAt(1));

  auto bound_output_sequence = winrt::single_threaded_vector<TensorInt64Bit>();
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"output_sequence", bound_output_sequence));
  WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L""));
  WINML_EXPECT_EQUAL(static_cast<uint32_t>(2), bound_output_sequence.Size());
  WINML_EXPECT_EQUAL(2, bound_output_sequence.GetAt(0).Shape().GetAt(0));
  WINML_EXPECT_EQUAL(3, bound_output_sequence.GetAt(0).Shape().GetAt(1));
  WINML_EXPECT_EQUAL(2, bound_output_sequence.GetAt(1).Shape().GetAt(0));
  WINML_EXPECT_EQUAL(3, bound_output_sequence.GetAt(1).Shape().GetAt(1));
}

const LearningModelBindingAPITestsApi& getapi() {
  static LearningModelBindingAPITestsApi api = {
    LearningModelBindingAPITestsClassSetup,
    CpuSqueezeNet,
    CpuSqueezeNetEmptyOutputs,
    CpuSqueezeNetUnboundOutputs,
    CpuSqueezeNetBindInputTensorAsInspectable,
    CastMapInt64,
    DictionaryVectorizerMapInt64,
    DictionaryVectorizerMapString,
    ZipMapInt64,
    ZipMapInt64Unbound,
    ZipMapString,
    GpuSqueezeNet,
    GpuSqueezeNetEmptyOutputs,
    GpuSqueezeNetUnboundOutputs,
    ImageBindingDimensions,
    VerifyInvalidBindExceptions,
    BindInvalidInputName,
    VerifyOutputAfterEvaluateAsyncCalledTwice,
    VerifyOutputAfterImageBindCalledTwice,
    SequenceLengthTensorFloat,
    SequenceConstructTensorString};

  if (SkipGpuTests()) {
    api.GpuSqueezeNet = SkipTest;
    api.GpuSqueezeNetEmptyOutputs = SkipTest;
    api.GpuSqueezeNetUnboundOutputs = SkipTest;
  }
  if (RuntimeParameterExists(L"noVideoFrameTests")) {
    api.ImageBindingDimensions = SkipTest;
    api.BindInvalidInputName = SkipTest;
    api.VerifyOutputAfterImageBindCalledTwice = SkipTest;
    api.VerifyInvalidBindExceptions = SkipTest;
  }
  return api;
}