mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-17 18:40:28 +00:00
Update C# Pages in view of the new preferred inference API (#17642)
### Description Re-work C# code samples with OrtValue API. Bring `C# Tutorial: Basic` to index. The site is currently published [here](https://yuslepukhin.github.io/onnxruntime) ### Motivation and Context Direct all future usage to `Ortvalue` API
This commit is contained in:
Dmitri Smirnov
GitHub
parent
acc1b8b5ea
commit
f3fa223ee8
7 changed files with 462 additions and 189 deletions
|
|
@ -16,7 +16,7 @@ nav_order: 4
|
|||
## Install the Nuget Packages with the .NET CLI
|
||||
|
||||
```bash
|
||||
dotnet add package Microsoft.ML.OnnxRuntime --version 1.2.0
|
||||
dotnet add package Microsoft.ML.OnnxRuntime --version 1.16.0
|
||||
dotnet add package System.Numerics.Tensors --version 0.1.0
|
||||
```
|
||||
|
||||
|
|
@ -42,28 +42,82 @@ This is an [Azure Function](https://azure.microsoft.com/services/functions/) exa
|
|||
|
||||
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
|
||||
dynamic data = JsonConvert.DeserializeObject(requestBody);
|
||||
review = review ?? data?.review;
|
||||
review ??= data.review;
|
||||
Debug.Assert(!string.IsNullOrEmpty(review), "Expecting a string with a content");
|
||||
|
||||
// Get path to model to create inference session.
|
||||
var modelPath = "./model.onnx";
|
||||
|
||||
// create input tensor (nlp example)
|
||||
var inputTensor = new DenseTensor<string>(new string[] { review }, new int[] { 1, 1 });
|
||||
|
||||
// Create input data for session.
|
||||
var input = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor<string>("input", inputTensor) };
|
||||
const string modelPath = "./model.onnx";
|
||||
|
||||
// Create an InferenceSession from the Model Path.
|
||||
var session = new InferenceSession(modelPath);
|
||||
// Creating and loading sessions are expensive per request.
|
||||
// They better be cached
|
||||
using var session = new InferenceSession(modelPath);
|
||||
|
||||
// Run session and send input data in to get inference output. Call ToList then get the Last item. Then use the AsEnumerable extension method to return the Value result as an Enumerable of NamedOnnxValue.
|
||||
var output = session.Run(input).ToList().Last().AsEnumerable<NamedOnnxValue>();
|
||||
// create input tensor (nlp example)
|
||||
using var inputOrtValue = OrtValue.CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, new long[] { 1, 1 });
|
||||
inputOrtValue.StringTensorSetElementAt(review, 0);
|
||||
|
||||
// Create input data for session. Request all outputs in this case.
|
||||
var inputs = new Dictionary<string, OrtValue>
|
||||
{
|
||||
{ "input", inputOrtValue }
|
||||
};
|
||||
|
||||
using var runOptions = new RunOptions();
|
||||
|
||||
// We are getting a sequence of maps as output. We are interested in the first element (map) of the sequence.
|
||||
// That result is a Sequence of Maps, and we only need the first map from there.
|
||||
using var outputs = session.Run(runOptions, inputs, session.OutputNames);
|
||||
Debug.Assert(outputs.Count > 0, "Expecting some output");
|
||||
|
||||
// We want the last output, which is the sequence of maps
|
||||
var lastOutput = outputs[outputs.Count - 1];
|
||||
|
||||
// Optional code to check the output type
|
||||
{
|
||||
var outputTypeInfo = lastOutput.GetTypeInfo();
|
||||
Debug.Assert(outputTypeInfo.OnnxType == OnnxValueType.ONNX_TYPE_SEQUENCE, "Expecting a sequence");
|
||||
|
||||
var sequenceTypeInfo = outputTypeInfo.SequenceTypeInfo;
|
||||
Debug.Assert(sequenceTypeInfo.ElementType.OnnxType == OnnxValueType.ONNX_TYPE_MAP, "Expecting a sequence of maps");
|
||||
}
|
||||
|
||||
var elementsNum = lastOutput.GetValueCount();
|
||||
Debug.Assert(elementsNum > 0, "Expecting a non empty sequence");
|
||||
|
||||
// Get the first map in sequence
|
||||
using var firstMap = lastOutput.GetValue(0, OrtAllocator.DefaultInstance);
|
||||
|
||||
// Optional code just checking
|
||||
{
|
||||
// Maps always have two elements, keys and values
|
||||
// We are expecting this to be a map of strings to floats
|
||||
var mapTypeInfo = firstMap.GetTypeInfo().MapTypeInfo;
|
||||
Debug.Assert(mapTypeInfo.KeyType == TensorElementType.String, "Expecting keys to be strings");
|
||||
Debug.Assert(mapTypeInfo.ValueType.OnnxType == OnnxValueType.ONNX_TYPE_TENSOR, "Values are in the tensor");
|
||||
Debug.Assert(mapTypeInfo.ValueType.TensorTypeAndShapeInfo.ElementDataType == TensorElementType.Float, "Result map value is float");
|
||||
}
|
||||
|
||||
var inferenceResult = new Dictionary<string, float>();
|
||||
// Let use the visitor to read map keys and values
|
||||
// Here keys and values are represented with the same number of corresponding entries
|
||||
// string -> float
|
||||
firstMap.ProcessMap((keys, values) => {
|
||||
// Access native buffer directly
|
||||
var valuesSpan = values.GetTensorDataAsSpan<float>();
|
||||
|
||||
var entryCount = (int)keys.GetTensorTypeAndShape().ElementCount;
|
||||
inferenceResult.EnsureCapacity(entryCount);
|
||||
for (int i = 0; i < entryCount; ++i)
|
||||
{
|
||||
inferenceResult.Add(keys.GetStringElement(i), valuesSpan[i]);
|
||||
}
|
||||
}, OrtAllocator.DefaultInstance);
|
||||
|
||||
// From the Enumerable output create the inferenceResult by getting the First value and using the AsDictionary extension method of the NamedOnnxValue.
|
||||
var inferenceResult = output.First().AsDictionary<string, float>();
|
||||
|
||||
// Return the inference result as json.
|
||||
return new JsonResult(inferenceResult);
|
||||
|
||||
}
|
||||
```
|
||||
## Reuse input/output tensor buffers
|
||||
|
|
@ -73,46 +127,80 @@ In some scenarios, you may want to reuse input/output tensors. This often happen
|
|||
### Chaining: Feed model A's output(s) as input(s) to model B
|
||||
|
||||
```cs
|
||||
InferenceSession session1, session2; // let's say 2 sessions are initialized
|
||||
using Microsoft.ML.OnnxRuntime.Tensors;
|
||||
using Microsoft.ML.OnnxRuntime;
|
||||
|
||||
Tensor<long> input = new DenseTensor<long>(new[] { 1, inputDimension }); // let's say data is fed into the Tensor objects
|
||||
var inputs1 = new List<NamedOnnxValue>()
|
||||
{
|
||||
NamedOnnxValue.CreateFromTensor("name1", input)
|
||||
};
|
||||
// session1 inference
|
||||
using (var outputs1 = session1.Run(inputs1))
|
||||
namespace Samples
|
||||
{
|
||||
// get intermediate value
|
||||
var input2 = outputs1.First();
|
||||
|
||||
// modify the name of the ONNX value
|
||||
input2.Name = "name2";
|
||||
|
||||
// create input list for session2
|
||||
var inputs2 = new List<NamedOnnxValue>() { input2 };
|
||||
|
||||
// session2 inference
|
||||
using (var results = session2.Run(inputs2))
|
||||
class FeedModelAToModelB
|
||||
{
|
||||
// manipulate the results
|
||||
static void Program()
|
||||
{
|
||||
const string modelAPath = "./modelA.onnx";
|
||||
const string modelBPath = "./modelB.onnx";
|
||||
using InferenceSession session1 = new InferenceSession(modelAPath);
|
||||
using InferenceSession session2 = new InferenceSession(modelBPath);
|
||||
|
||||
// Illustration only
|
||||
float[] inputData = { 1, 2, 3, 4 };
|
||||
long[] inputShape = { 1, 4 };
|
||||
|
||||
using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(inputData, inputShape);
|
||||
|
||||
// Create input data for session. Request all outputs in this case.
|
||||
var inputs1 = new Dictionary<string, OrtValue>
|
||||
{
|
||||
{ "input", inputOrtValue }
|
||||
};
|
||||
|
||||
using var runOptions = new RunOptions();
|
||||
|
||||
// session1 inference
|
||||
using (var outputs1 = session1.Run(runOptions, inputs1, session1.OutputNames))
|
||||
{
|
||||
// get intermediate value
|
||||
var outputToFeed = outputs1.First();
|
||||
|
||||
// modify the name of the ONNX value
|
||||
// create input list for session2
|
||||
var inputs2 = new Dictionary<string, OrtValue>
|
||||
{
|
||||
{ "inputNameForModelB", outputToFeed }
|
||||
};
|
||||
|
||||
// session2 inference
|
||||
using (var results = session2.Run(runOptions, inputs2, session2.OutputNames))
|
||||
{
|
||||
// manipulate the results
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
### Multiple inference runs with fixed sized input(s) and output(s)
|
||||
|
||||
If the model have fixed sized inputs and outputs of numeric tensors, you can use "FixedBufferOnnxValue" to accelerate the inference speed. By using "FixedBufferOnnxValue", the container objects only need to be allocated/disposed one time during multiple InferenceSession.Run() calls. This avoids some overhead which may be beneficial for smaller models where the time is noticeable in the overall running time.
|
||||
If the model have fixed sized inputs and outputs of numeric tensors,
|
||||
use the preferable **OrtValue** and its API to accelerate the inference speed and minimize data transfer.
|
||||
**OrtValue** class makes it possible to reuse the underlying buffer for the input and output tensors.
|
||||
It pins the managed buffers and makes use of them for inference. It also provides direct access
|
||||
to the native buffers for outputs. You can also preallocate `OrtValue` for outputs or create it on top
|
||||
of the existing buffers.
|
||||
This avoids some overhead which may be beneficial for smaller models
|
||||
where the time is noticeable in the overall running time.
|
||||
|
||||
<!-- FIXME!: This test is no longer in the repo. Needs to be fixed. -->
|
||||
<!-- An example can be found at `TestReusingFixedBufferOnnxValueNonStringTypeMultiInferences()`:
|
||||
* [Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs#L1047](https://github.com/microsoft/onnxruntime/blob/main/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs#L1047) -->
|
||||
Keep in mind that **OrtValue** class, like many other classes in Onnruntime C# API is **IDisposable**.
|
||||
It needs to be properly disposed to either unpin the managed buffers or release the native buffers
|
||||
to avoid memory leaks.
|
||||
|
||||
## Running on GPU (Optional)
|
||||
If using the GPU package, simply use the appropriate SessionOptions when creating an InferenceSession.
|
||||
|
||||
```cs
|
||||
int gpuDeviceId = 0; // The GPU device ID to execute on
|
||||
var session = new InferenceSession("model.onnx", SessionOptions.MakeSessionOptionWithCudaProvider(gpuDeviceId));
|
||||
using var gpuSessionOptoins = SessionOptions.MakeSessionOptionWithCudaProvider(gpuDeviceId);
|
||||
using var session = new InferenceSession("model.onnx", gpuSessionOptoins);
|
||||
```
|
||||
# ONNX Runtime C# API
|
||||
{: .no_toc }
|
||||
|
|
|
|||
|
|
@ -1,36 +1,150 @@
|
|||
---
|
||||
nav_exclude: true
|
||||
title: Basic C# Tutorial
|
||||
description: Basic usage of C# API
|
||||
parent: Inference with C#
|
||||
grand_parent: Tutorials
|
||||
has_children: false
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
|
||||
# C# Tutorial: Basic
|
||||
|
||||
Here is simple tutorial for getting started with running inference on an existing ONNX model for a given input data. The model is typically trained using any of the well-known training frameworks and exported into the ONNX format.
|
||||
Here is a simple tutorial for getting started with running inference on an existing ONNX model for a given input data.
|
||||
The model is typically trained using any of the well-known training frameworks and then exported into the ONNX format.
|
||||
|
||||
Note, that the following classes `NamedOnnxValue`, `DisposableNamedOnnxValue`, `FixedBufferOnnxValue` are going
|
||||
to be deprecated in the future. They are not recommended for new code.
|
||||
|
||||
The new `OrtValue` based API is the recommended approach. The `OrtValue` API generates less garbage and is more performant.
|
||||
Some scenarios indicated 4x performance improvement over the previous API and significantly less garbage.
|
||||
It provides uniform access to data via `ReadOnlySpan<T>` and `Span<T>` structures, regardless of its location, managed or unmanaged.
|
||||
|
||||
`DenseTensor` class can still be used for multi-dimensional access to the data since the new `Span` based API feature
|
||||
only a 1-D index. However, some reported a slow performance when using `DenseTensor` class multi-dimensional access.
|
||||
One can then create an OrtValue on top of the tensors data.
|
||||
|
||||
`ShapeUtils` class provides some help to deal with multi-dimensional indices for OrtValues.
|
||||
|
||||
`OrtValue` based API provides direct native memory access in a type safe manner using `ReadOnlySpan<T>` and `Span<T>` stack bases structures.
|
||||
OrtValue is a universal container that can hold different ONNX types, such as tensors, maps, and sequences.
|
||||
It always existed in the onnxruntime library, but was not exposed in the C# API.
|
||||
|
||||
As before, `OrtValues` can be created directly on top of the managed `unmanaged` (struct based blittable types) arrays.
|
||||
Read MS documentation on `blittable` data types. onnxruntime C# API allows use of managed buffers for input or output.
|
||||
|
||||
If output shapes are known, one can pre-allocate `OrtValue` on top of the managed or unmanaged allocations and supply
|
||||
those OrtValues to be used as outputs. Due to this fact, the need for `IOBinding` is greatly diminished.
|
||||
|
||||
String data is represented as UTF-16 string objects in C#. It will still need to be copied and converted to UTF-8 to the native
|
||||
memory. However, that conversion is now more optimized and is done in a single pass without intermediate byte arrays.
|
||||
The same applies to string `OrtValue` tensors returned as outputs. Character based API now operates on `Span<char>`,
|
||||
`ReadOnlySpan<char>`, and `ReadOnlyMemory<char>` objects. This adds flexibility to the API and allows to avoid unnecessary copies.
|
||||
|
||||
Except some of the above deprecated API classes, nearly all of C# API classes are `IDisposable`.
|
||||
Meaning they need to be disposed after use, otherwise you will get memory leaks.
|
||||
Because OrtValues are used to hold tensor data, the sizes of the leaks can be huge. They are likely
|
||||
to accumulate with each `Run` call, as each inference call requires input OrtValues and returns output OrtValues.
|
||||
Do not hold your breath for finalizers which are not guaranteed to ever run, and if they do, they do it
|
||||
when it is too late.
|
||||
|
||||
This includes `SessionOptions`, `RunOptions`, `InferenceSession`, `OrtValue`. Run() calls return `IDisposableCollection`
|
||||
that allows to dispose all of the containing objects in one statement or `using`. This is because these objects
|
||||
own some native resource, often a native object.
|
||||
|
||||
Not disposing `OrtValue` that was created on top of the managed buffer would result in
|
||||
that buffer pinned in memory indefinitely. Such a buffer can not be garbage collected or moved in memory.
|
||||
|
||||
`OrtValue`s that were created on top of the native onnxruntime memory should also be disposed of promptly.
|
||||
Otherwise, the native memory will not be deallocated. OrtValues returned by `Run()` usually hold native memory.
|
||||
|
||||
GC can not operate on native memory or any other native resources.
|
||||
|
||||
The `using` statement or a block is a convenient way to ensure that the objects are disposed.
|
||||
`InferenceSession` can be a long lived object and a member of another class. It eventually must also need to be disposed.
|
||||
This means, the containing class also would have to be made disposable to achieve this.
|
||||
|
||||
OrtValue API also provides visitor like API to walk ONNX maps and sequences.
|
||||
This is a more efficient way to access Onnxruntime data.
|
||||
|
||||
To start scoring using the model, open a session using the `InferenceSession` class, passing in the file path to the model as a parameter.
|
||||
|
||||
|
||||
```cs
|
||||
var session = new InferenceSession("model.onnx");
|
||||
using var session = new InferenceSession("model.onnx");
|
||||
```
|
||||
|
||||
Once a session is created, you can execute queries using the `Run` method of the `InferenceSession` object. Currently, only `Tensor` type of input and outputs are supported. The results of the `Run` method are represented as a collection of .Net `Tensor` objects (as defined in [System.Numerics.Tensor](https://www.nuget.org/packages/System.Numerics.Tensors)).
|
||||
Once a session is created, you can execute queries using the `Run` method of the `InferenceSession` object.
|
||||
|
||||
```cs
|
||||
Tensor<float> t1, t2; // let's say data is fed into the Tensor objects
|
||||
var inputs = new List<NamedOnnxValue>()
|
||||
{
|
||||
NamedOnnxValue.CreateFromTensor<float>("name1", t1),
|
||||
NamedOnnxValue.CreateFromTensor<float>("name2", t2)
|
||||
};
|
||||
using (var results = session.Run(inputs))
|
||||
{
|
||||
// manipulate the results
|
||||
}
|
||||
```
|
||||
|
||||
You can load your input data into Tensor<T> objects in several ways. A simple example is to create the Tensor from arrays.
|
||||
|
||||
```cs
|
||||
float[] sourceData; // assume your data is loaded into a flat float array
|
||||
int[] dimensions; // and the dimensions of the input is stored here
|
||||
Tensor<float> t1 = new DenseTensor<float>(sourceData, dimensions);
|
||||
long[] dimensions; // and the dimensions of the input is stored here
|
||||
|
||||
// Create a OrtValue on top of the sourceData array
|
||||
using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(sourceData, dimensions);
|
||||
|
||||
var inputs = new Dictionary<string, OrtValue> {
|
||||
{ "name1", inputOrtValue }
|
||||
};
|
||||
|
||||
|
||||
using var runOptions = new RunOptions();
|
||||
|
||||
// Pass inputs and request the first output
|
||||
// Note that the output is a disposable collection that holds OrtValues
|
||||
using var output = session.Run(runOptions, inputs, session.OutputNames[0]);
|
||||
|
||||
var output_0 = output[0];
|
||||
|
||||
// Assuming the output contains a tensor of float data, you can access it as follows
|
||||
// Returns Span<float> which points directly to native memory.
|
||||
var outputData = output_0.GetTensorDataAsSpan<float>();
|
||||
|
||||
// If you are interested in more information about output, request its type and shape
|
||||
// Assuming it is a tensor
|
||||
// This is not disposable, will be GCed
|
||||
// There you can request Shape, ElementDataType, etc
|
||||
var tensorTypeAndShape = output_0.GetTensorTypeAndShape();
|
||||
|
||||
```
|
||||
You can still use `Tensor` class for data manipulation if you have existing code that does it.
|
||||
Then create `OrtValue` on top of Tensor buffer.
|
||||
|
||||
```cs
|
||||
|
||||
// Create and manipulate the data using tensor interface
|
||||
DenseTensor<float> t1 = new DenseTensor<float>(sourceData, dimensions);
|
||||
|
||||
// One minor inconvenience is that Tensor class operates on `int` dimensions and indices.
|
||||
// OrtValue dimensions are `long`. This is required, because `OrtValue` talks directly to
|
||||
// Ort API and the library uses long dimensions.
|
||||
|
||||
// Convert dims to long[]
|
||||
var shape = Array.Convert<int,long>(dimensions, Convert.ToInt64);
|
||||
|
||||
using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance,
|
||||
t1.Buffer, shape);
|
||||
|
||||
```
|
||||
|
||||
Here is a way to populate a string tensor. Strings can not be mapped, and must be copy/converted to native memory.
|
||||
To that end we pre-allocate a native tensor of empty strings with specified dimensions, and then
|
||||
set individual strings by index.
|
||||
|
||||
|
||||
```cs
|
||||
|
||||
string[] strs = { "Hello", "Ort", "World" };
|
||||
long[] shape = { 1, 1, 3 };
|
||||
var elementsNum = ShapeUtils.GetSizeForShape(shape);
|
||||
|
||||
using var strTensor = OrtValue.CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, shape);
|
||||
|
||||
for (long i = 0; i < elementsNum; ++i)
|
||||
{
|
||||
strTensor.StringTensorSetElementAt(strs[i].AsSpan(), i);
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -136,8 +136,8 @@ Now that we have tested the model in Python its time to build it out in C#. The
|
|||
### Install the Nuget Packages
|
||||
- Install the Nuget packages `BERTTokenizers`, `Microsoft.ML.OnnxRuntime`, `Microsoft.ML.OnnxRuntime.Managed`, `Microsoft.ML`
|
||||
```PowerShell
|
||||
dotnet add package Microsoft.ML.OnnxRuntime --version 1.12.0
|
||||
dotnet add package Microsoft.ML.OnnxRuntime.Managed --version 1.12.0
|
||||
dotnet add package Microsoft.ML.OnnxRuntime --version 1.16.0
|
||||
dotnet add package Microsoft.ML.OnnxRuntime.Managed --version 1.16.0
|
||||
dotnet add package dotnet add package Microsoft.ML
|
||||
dotnet add package dotnet add package BERTTokenizers --version 1.1.0
|
||||
```
|
||||
|
|
@ -159,7 +159,7 @@ using System;
|
|||
|
||||
namespace MyApp // Note: actual namespace depends on the project name.
|
||||
{
|
||||
internal class Program
|
||||
internal class BertTokenizeProgram
|
||||
{
|
||||
static void Main(string[] args)
|
||||
{
|
||||
|
|
@ -169,10 +169,10 @@ namespace MyApp // Note: actual namespace depends on the project name.
|
|||
}
|
||||
```
|
||||
### Create the BertInput class for encoding
|
||||
- Add the `BertInput` class
|
||||
- Add the `BertInput` struct
|
||||
|
||||
```csharp
|
||||
public class BertInput
|
||||
public struct BertInput
|
||||
{
|
||||
public long[] InputIds { get; set; }
|
||||
public long[] AttentionMask { get; set; }
|
||||
|
|
@ -205,83 +205,86 @@ namespace MyApp // Note: actual namespace depends on the project name.
|
|||
};
|
||||
|
||||
```
|
||||
### Create the Tensors
|
||||
- Create the `ConvertToTensor` function. Set the shape of the Tensor `new[] { 1, inputDimension }` and the values to be added to the `NamedOnnxValue` input list.
|
||||
### Create the `inputs` of `name -> OrtValue` pairs as required for inference
|
||||
|
||||
```csharp
|
||||
public static Tensor<long> ConvertToTensor(long[] inputArray, int inputDimension)
|
||||
{
|
||||
// Create a tensor with the shape the model is expecting. Here we are sending in 1 batch with the inputDimension as the amount of tokens.
|
||||
Tensor<long> input = new DenseTensor<long>(new[] { 1, inputDimension });
|
||||
|
||||
// Loop through the inputArray (InputIds, AttentionMask and TypeIds)
|
||||
for (var i = 0; i < inputArray.Length; i++)
|
||||
{
|
||||
// Add each to the input Tenor result.
|
||||
// Set index and array value of each input Tensor.
|
||||
input[0,i] = inputArray[i];
|
||||
}
|
||||
return input;
|
||||
}
|
||||
```
|
||||
|
||||
### Create the `input` of `List<NamedOnnxValue>` that is needed for inference
|
||||
|
||||
- Get the model, call the `ConvertToTensor` function to create the tensor and create the list of `NamedOnnxValue` input variables for inferencing.
|
||||
- Get the model, create 3 OrtValues on top of the input buffers and wrap them into a Dictionary to feed into a Run().
|
||||
Beware that almost all of the Onnxruntime classes wrap native data structures, and, therefore, must be disposed
|
||||
to prevent memory leaks.
|
||||
|
||||
```csharp
|
||||
// Get path to model to create inference session.
|
||||
var modelPath = @"C:\code\bert-nlp-csharp\BertNlpTest\BertNlpTest\bert-large-uncased-finetuned-qa.onnx";
|
||||
|
||||
// Create input tensor.
|
||||
using var runOptions = new RunOptions();
|
||||
using var session = new InferenceSession(modelPath);
|
||||
|
||||
var input_ids = ConvertToTensor(bertInput.InputIds, bertInput.InputIds.Length);
|
||||
var attention_mask = ConvertToTensor(bertInput.AttentionMask, bertInput.InputIds.Length);
|
||||
var token_type_ids = ConvertToTensor(bertInput.TypeIds, bertInput.InputIds.Length);
|
||||
// Create input tensors over the input data.
|
||||
using var inputIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.InputIds,
|
||||
new long[] { 1, bertInput.InputIds.Length });
|
||||
|
||||
using var attMaskOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.AttentionMask,
|
||||
new long[] { 1, bertInput.AttentionMask.Length });
|
||||
|
||||
// Create input data for session.
|
||||
var input = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor("input_ids", input_ids),
|
||||
NamedOnnxValue.CreateFromTensor("input_mask", attention_mask),
|
||||
NamedOnnxValue.CreateFromTensor("segment_ids", token_type_ids) };
|
||||
using var typeIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.TypeIds,
|
||||
new long[] { 1, bertInput.TypeIds.Length });
|
||||
|
||||
// Create input data for session. Request all outputs in this case.
|
||||
var inputs = new Dictionary<string, OrtValue>
|
||||
{
|
||||
{ "input_ids", inputIdsOrtValue },
|
||||
{ "input_mask", attMaskOrtValue },
|
||||
{ "segment_ids", typeIdsOrtValue }
|
||||
};
|
||||
|
||||
```
|
||||
### Run Inference
|
||||
- Create the `InferenceSession`, run the inference and print out the result.
|
||||
|
||||
```csharp
|
||||
// Create an InferenceSession from the Model Path.
|
||||
var session = new InferenceSession(modelPath);
|
||||
|
||||
// Run session and send the input data in to get inference output.
|
||||
var output = session.Run(input);
|
||||
using var output = session.Run(runOptions, inputs, session.OutputNames);
|
||||
```
|
||||
### Postprocess the `output` and print the result
|
||||
|
||||
- Here we get the index for the start position (`startLogit`) and end position (`endLogits`). Then we take the original `tokens` of the input sentence and get the vocabulary value for the token ids predicted.
|
||||
|
||||
```csharp
|
||||
// Call ToList on the output.
|
||||
// Get the First and Last item in the list.
|
||||
// Get the Value of the item and cast as IEnumerable<float> to get a list result.
|
||||
List<float> startLogits = (output.ToList().First().Value as IEnumerable<float>).ToList();
|
||||
List<float> endLogits = (output.ToList().Last().Value as IEnumerable<float>).ToList();
|
||||
// Get the Index of the Max value from the output lists.
|
||||
// We intentionally do not copy to an array or to a list to employ algorithms.
|
||||
// Hopefully, more algos will be available in the future for spans.
|
||||
// so we can directly read from native memory and do not duplicate data that
|
||||
// can be large for some models
|
||||
// Local function
|
||||
int GetMaxValueIndex(ReadOnlySpan<float> span)
|
||||
{
|
||||
float maxVal = span[0];
|
||||
int maxIndex = 0;
|
||||
for (int i = 1; i < span.Length; ++i)
|
||||
{
|
||||
var v = span[i];
|
||||
if (v > maxVal)
|
||||
{
|
||||
maxVal = v;
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
return maxIndex;
|
||||
}
|
||||
|
||||
// Get the Index of the Max value from the output lists.
|
||||
var startIndex = startLogits.ToList().IndexOf(startLogits.Max());
|
||||
var endIndex = endLogits.ToList().IndexOf(endLogits.Max());
|
||||
var startLogits = output[0].GetTensorDataAsSpan<float>();
|
||||
int startIndex = GetMaxValueIndex(startLogits);
|
||||
|
||||
// From the list of the original tokens in the sentence
|
||||
// Get the tokens between the startIndex and endIndex and convert to the vocabulary from the ID of the token.
|
||||
var predictedTokens = tokens
|
||||
.Skip(startIndex)
|
||||
.Take(endIndex + 1 - startIndex)
|
||||
.Select(o => tokenizer.IdToToken((int)o.VocabularyIndex))
|
||||
.ToList();
|
||||
var endLogits = output[output.Count - 1].GetTensorDataAsSpan<float>();
|
||||
int endIndex = GetMaxValueIndex(endLogits);
|
||||
|
||||
// Print the result.
|
||||
Console.WriteLine(String.Join(" ", predictedTokens));
|
||||
var predictedTokens = tokens
|
||||
.Skip(startIndex)
|
||||
.Take(endIndex + 1 - startIndex)
|
||||
.Select(o => tokenizer.IdToToken((int)o.VocabularyIndex))
|
||||
.ToList();
|
||||
|
||||
// Print the result.
|
||||
Console.WriteLine(String.Join(" ", predictedTokens));
|
||||
```
|
||||
|
||||
## Deploy with Azure Web App
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ See this table for supported versions:
|
|||
|
||||
| ONNX Runtime Version | CUDA Toolkit Version | cuDNN Version|
|
||||
|----------------------|----------------------|--------------|
|
||||
| 1.13 - 1.14 | 11.6 | 8.5.0.96 |
|
||||
| 1.13 - 1.16 | 11.6 | 8.5.0.96 |
|
||||
| 1.9 - 1.12 | 11.4 | 8.2.2.26 |
|
||||
|
||||
NOTE: Full table can be found [here](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements)
|
||||
|
|
@ -47,7 +47,9 @@ torch.cuda.is_available()
|
|||
- Now you can enable GPU in the C# ONNX Runtime API with the following code:
|
||||
|
||||
```cs
|
||||
var session = new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(0));
|
||||
// keep in mind almost all of the classes are disposable.
|
||||
using var gpuSessionOptions = SessionOptions.MakeSessionOptionWithCudaProvider(0);
|
||||
using var session = new InferenceSession(modelPath, gpuSessionOptions);
|
||||
```
|
||||
|
||||
## Checkout more C# ONNX Runtime resources
|
||||
|
|
|
|||
|
|
@ -73,8 +73,11 @@ Next, we will preprocess the image according to the [requirements of the model](
|
|||
```cs
|
||||
var paddedHeight = (int)(Math.Ceiling(image.Height / 32f) * 32f);
|
||||
var paddedWidth = (int)(Math.Ceiling(image.Width / 32f) * 32f);
|
||||
Tensor<float> input = new DenseTensor<float>(new[] { 3, paddedHeight, paddedWidth });
|
||||
var mean = new[] { 102.9801f, 115.9465f, 122.7717f };
|
||||
|
||||
// Preprocessing image
|
||||
// We use DenseTensor for multi-dimensional access
|
||||
DenseTensor<float> input = new(new[] { 3, paddedHeight, paddedWidth });
|
||||
image.ProcessPixelRows(accessor =>
|
||||
{
|
||||
for (int y = paddedHeight - accessor.Height; y < accessor.Height; y++)
|
||||
|
|
@ -92,15 +95,26 @@ image.ProcessPixelRows(accessor =>
|
|||
|
||||
Here, we're creating a Tensor of the required size `(channels, paddedHeight, paddedWidth)`, accessing the pixel values, preprocessing them and finally assigning them to the tensor at the appropriate indicies.
|
||||
|
||||
|
||||
### Setup inputs
|
||||
|
||||
// Pin DenseTensor memory and use it directly in the OrtValue tensor
|
||||
// It will be unpinned on ortValue disposal
|
||||
|
||||
```cs
|
||||
using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance,
|
||||
input.Buffer, new long[] { 3, paddedHeight, paddedWidth });
|
||||
```
|
||||
|
||||
Next, we will create the inputs to the model:
|
||||
|
||||
```cs
|
||||
var inputs = new List<NamedOnnxValue>
|
||||
|
||||
var inputs = new Dictionary<string, OrtValue>
|
||||
{
|
||||
NamedOnnxValue.CreateFromTensor("image", input)
|
||||
{ "image", inputOrtValue }
|
||||
};
|
||||
|
||||
```
|
||||
|
||||
To check the input node names for an ONNX model, you can use [Netron](https://github.com/lutzroeder/netron) to visualise the model and see input/output names. In this case, this model has `image` as the input node name.
|
||||
|
|
@ -111,7 +125,9 @@ Next, we will create an inference session and run the input through it:
|
|||
|
||||
```cs
|
||||
using var session = new InferenceSession(modelFilePath);
|
||||
using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.Run(inputs);
|
||||
using var runOptions = new RunOptions();
|
||||
using IDisposableReadOnlyCollection<OrtValue> results = session.Run(runOptions, inputs, session.OutputNames);
|
||||
|
||||
```
|
||||
|
||||
### Postprocess output
|
||||
|
|
@ -119,22 +135,23 @@ using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.
|
|||
Next, we will need to postprocess the output to get boxes and associated label and confidence scores for each box:
|
||||
|
||||
```cs
|
||||
var resultsArray = results.ToArray();
|
||||
float[] boxes = resultsArray[0].AsEnumerable<float>().ToArray();
|
||||
long[] labels = resultsArray[1].AsEnumerable<long>().ToArray();
|
||||
float[] confidences = resultsArray[2].AsEnumerable<float>().ToArray();
|
||||
var boxesSpan = results[0].GetTensorDataAsSpan<float>();
|
||||
var labelsSpan = results[1].GetTensorDataAsSpan<long>();
|
||||
var confidencesSpan = results[2].GetTensorDataAsSpan<float>();
|
||||
|
||||
const float minConfidence = 0.7f;
|
||||
var predictions = new List<Prediction>();
|
||||
var minConfidence = 0.7f;
|
||||
for (int i = 0; i < boxes.Length - 4; i += 4)
|
||||
|
||||
for (int i = 0; i < boxesSpan.Length - 4; i += 4)
|
||||
{
|
||||
var index = i / 4;
|
||||
if (confidences[index] >= minConfidence)
|
||||
if (confidencesSpan[index] >= minConfidence)
|
||||
{
|
||||
predictions.Add(new Prediction
|
||||
{
|
||||
Box = new Box(boxes[i], boxes[i + 1], boxes[i + 2], boxes[i + 3]),
|
||||
Label = LabelMap.Labels[labels[index]],
|
||||
Confidence = confidences[index]
|
||||
Box = new Box(boxesSpan[i], boxesSpan[i + 1], boxesSpan[i + 2], boxesSpan[i + 3]),
|
||||
Label = LabelMap.Labels[labelsSpan[index]],
|
||||
Confidence = confidencesSpan[index]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,9 +77,10 @@ Note, we're doing a centered crop resize to preserve aspect ratio.
|
|||
Next, we will preprocess the image according to the [requirements of the model](https://github.com/onnx/models/tree/master/vision/classification/resnet#preprocessing):
|
||||
|
||||
```cs
|
||||
Tensor<float> input = new DenseTensor<float>(new[] { 1, 3, 224, 224 });
|
||||
// We use DenseTensor for multi-dimensional access to populate the image data
|
||||
var mean = new[] { 0.485f, 0.456f, 0.406f };
|
||||
var stddev = new[] { 0.229f, 0.224f, 0.225f };
|
||||
DenseTensor<float> processedImage = new(new[] { 1, 3, 224, 224 });
|
||||
image.ProcessPixelRows(accessor =>
|
||||
{
|
||||
for (int y = 0; y < accessor.Height; y++)
|
||||
|
|
@ -87,9 +88,9 @@ image.ProcessPixelRows(accessor =>
|
|||
Span<Rgb24> pixelSpan = accessor.GetRowSpan(y);
|
||||
for (int x = 0; x < accessor.Width; x++)
|
||||
{
|
||||
input[0, 0, y, x] = ((pixelSpan[x].R / 255f) - mean[0]) / stddev[0];
|
||||
input[0, 1, y, x] = ((pixelSpan[x].G / 255f) - mean[1]) / stddev[1];
|
||||
input[0, 2, y, x] = ((pixelSpan[x].B / 255f) - mean[2]) / stddev[2];
|
||||
processedImage[0, 0, y, x] = ((pixelSpan[x].R / 255f) - mean[0]) / stddev[0];
|
||||
processedImage[0, 1, y, x] = ((pixelSpan[x].G / 255f) - mean[1]) / stddev[1];
|
||||
processedImage[0, 2, y, x] = ((pixelSpan[x].B / 255f) - mean[2]) / stddev[2];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -102,10 +103,17 @@ Here, we're creating a Tensor of the required size `(batch-size, channels, heigh
|
|||
Next, we will create the inputs to the model:
|
||||
|
||||
```cs
|
||||
var inputs = new List<NamedOnnxValue>
|
||||
// Pin tensor buffer and create a OrtValue with native tensor that makes use of
|
||||
// DenseTensor buffer directly. This avoids extra data copy within OnnxRuntime.
|
||||
// It will be unpinned on ortValue disposal
|
||||
using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance,
|
||||
processedImage.Buffer, new long[] { 1, 3, 224, 224 });
|
||||
|
||||
var inputs = new Dictionary<string, OrtValue>
|
||||
{
|
||||
NamedOnnxValue.CreateFromTensor("data", input)
|
||||
};
|
||||
{ "data", inputOrtValue }
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
To check the input node names for an ONNX model, you can use [Netron](https://github.com/lutzroeder/netron) to visualise the model and see input/output names. In this case, this model has `data` as the input node name.
|
||||
|
|
@ -116,7 +124,8 @@ Next, we will create an inference session and run the input through it:
|
|||
|
||||
```cs
|
||||
using var session = new InferenceSession(modelFilePath);
|
||||
using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.Run(inputs);
|
||||
using var runOptions = new RunOptions();
|
||||
using IDisposableReadOnlyCollection<OrtValue> results = session.Run(runOptions, inputs, session.OutputNames);
|
||||
```
|
||||
|
||||
### Postprocess output
|
||||
|
|
@ -124,7 +133,9 @@ using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.
|
|||
Next, we will need to postprocess the output to get the softmax vector, as this is not handled by the model itself:
|
||||
|
||||
```cs
|
||||
IEnumerable<float> output = results.First().AsEnumerable<float>();
|
||||
// We copy results to array only to apply algorithms, otherwise data can be accessed directly
|
||||
// from the native buffer via ReadOnlySpan<T> or Span<T>
|
||||
var output = results[0].GetTensorDataAsSpan<float>().ToArray();
|
||||
float sum = output.Sum(x => (float)Math.Exp(x));
|
||||
IEnumerable<float> softmax = output.Select(x => (float)Math.Exp(x) / sum);
|
||||
```
|
||||
|
|
|
|||
|
|
@ -136,33 +136,51 @@ make a picture of green tree with flowers aroundit and a red sky
|
|||
```csharp
|
||||
public static int[] TokenizeText(string text)
|
||||
{
|
||||
// Create Tokenizer and tokenize the sentence.
|
||||
var tokenizerOnnxPath = Directory.GetCurrentDirectory().ToString() + ("\\text_tokenizer\\custom_op_cliptok.onnx");
|
||||
// Create Tokenizer and tokenize the sentence.
|
||||
var tokenizerOnnxPath = Directory.GetCurrentDirectory().ToString() + ("\\text_tokenizer\\custom_op_cliptok.onnx");
|
||||
|
||||
// Create session options for custom op of extensions
|
||||
var sessionOptions = new SessionOptions();
|
||||
var customOp = "ortextensions.dll";
|
||||
sessionOptions.RegisterCustomOpLibraryV2(customOp, out var libraryHandle);
|
||||
|
||||
// Create an InferenceSession from the onnx clip tokenizer.
|
||||
var tokenizeSession = new InferenceSession(tokenizerOnnxPath, sessionOptions);
|
||||
var inputTensor = new DenseTensor<string>(new string[] { text }, new int[] { 1 });
|
||||
var inputString = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor<string>("string_input", inputTensor) };
|
||||
// Run session and send the input data in to get inference output.
|
||||
var tokens = tokenizeSession.Run(inputString);
|
||||
var inputIds = (tokens.ToList().First().Value as IEnumerable<long>).ToArray();
|
||||
Console.WriteLine(String.Join(" ", inputIds));
|
||||
// Cast inputIds to Int32
|
||||
var InputIdsInt = inputIds.Select(x => (int)x).ToArray();
|
||||
var modelMaxLength = 77;
|
||||
// Pad array with 49407 until length is modelMaxLength
|
||||
if (InputIdsInt.Length < modelMaxLength)
|
||||
{
|
||||
var pad = Enumerable.Repeat(49407, 77 - InputIdsInt.Length).ToArray();
|
||||
InputIdsInt = InputIdsInt.Concat(pad).ToArray();
|
||||
}
|
||||
return InputIdsInt;
|
||||
// Create session options for custom op of extensions
|
||||
using var sessionOptions = new SessionOptions();
|
||||
var customOp = "ortextensions.dll";
|
||||
sessionOptions.RegisterCustomOpLibraryV2(customOp, out var libraryHandle);
|
||||
|
||||
// Create an InferenceSession from the onnx clip tokenizer.
|
||||
using var tokenizeSession = new InferenceSession(tokenizerOnnxPath, sessionOptions);
|
||||
|
||||
// Create input tensor from text
|
||||
using var inputTensor = OrtValue.CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, new long[] { 1 });
|
||||
inputTensor.StringTensorSetElementAt(text.AsSpan(), 0);
|
||||
|
||||
var inputs = new Dictionary<string, OrtValue>
|
||||
{
|
||||
{ "string_input", inputTensor }
|
||||
};
|
||||
|
||||
// Run session and send the input data in to get inference output.
|
||||
using var runOptions = new RunOptions();
|
||||
using var tokens = tokenizeSession.Run(runOptions, inputs, tokenizeSession.OutputNames);
|
||||
|
||||
var inputIds = tokens[0].GetTensorDataAsSpan<long>();
|
||||
|
||||
// Cast inputIds to Int32
|
||||
var InputIdsInt = new int[inputIds.Length];
|
||||
for(int i = 0; i < inputIds.Length; i++)
|
||||
{
|
||||
InputIdsInt[i] = (int)inputIds[i];
|
||||
}
|
||||
|
||||
Console.WriteLine(String.Join(" ", InputIdsInt));
|
||||
|
||||
var modelMaxLength = 77;
|
||||
// Pad array with 49407 until length is modelMaxLength
|
||||
if (InputIdsInt.Length < modelMaxLength)
|
||||
{
|
||||
var pad = Enumerable.Repeat(49407, 77 - InputIdsInt.Length).ToArray();
|
||||
InputIdsInt = InputIdsInt.Concat(pad).ToArray();
|
||||
}
|
||||
return InputIdsInt;
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
```text
|
||||
|
|
@ -186,26 +204,35 @@ The text encoder creates the text embedding which is trained to encode the text
|
|||
- Text Embedding: A vector of numbers that represents the text prompt created from the tokenization result. The text embedding is created by the `text_encoder` model.
|
||||
|
||||
```csharp
|
||||
public static DenseTensor<float> TextEncoder(int[] tokenizedInput)
|
||||
{
|
||||
// Create input tensor.
|
||||
var input_ids = TensorHelper.CreateTensor(tokenizedInput, new[] { 1, tokenizedInput.Count() });
|
||||
public static float[] TextEncoder(int[] tokenizedInput)
|
||||
{
|
||||
// Create input tensor. OrtValue will not copy, will read from managed memory
|
||||
using var input_ids = OrtValue.CreateTensorValueFromMemory<int>(tokenizedInput,
|
||||
new long[] { 1, tokenizedInput.Count() });
|
||||
|
||||
var input = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor<int>("input_ids", input_ids) };
|
||||
var textEncoderOnnxPath = Directory.GetCurrentDirectory().ToString() + ("\\text_encoder\\model.onnx");
|
||||
|
||||
var textEncoderOnnxPath = Directory.GetCurrentDirectory().ToString() + ("\\text_encoder\\model.onnx");
|
||||
using var encodeSession = new InferenceSession(textEncoderOnnxPath);
|
||||
|
||||
var encodeSession = new InferenceSession(textEncoderOnnxPath);
|
||||
// Run inference.
|
||||
var encoded = encodeSession.Run(input);
|
||||
// Pre-allocate the output so it goes to a managed buffer
|
||||
// we know the shape
|
||||
var lastHiddenState = new float[1 * 77 * 768];
|
||||
using var outputOrtValue = OrtValue.CreateTensorValueFromMemory<float>(lastHiddenState, new long[] { 1, 77, 768 });
|
||||
|
||||
var lastHiddenState = (encoded.ToList().First().Value as IEnumerable<float>).ToArray();
|
||||
var lastHiddenStateTensor = TensorHelper.CreateTensor(lastHiddenState.ToArray(), new[] { 1, 77, 768 });
|
||||
string[] input_names = { "input_ids" };
|
||||
OrtValue[] inputs = { input_ids };
|
||||
|
||||
return lastHiddenStateTensor;
|
||||
string[] output_names = { encodeSession.OutputNames[0] };
|
||||
OrtValue[] outputs = { outputOrtValue };
|
||||
|
||||
}
|
||||
// Run inference.
|
||||
using var runOptions = new RunOptions();
|
||||
encodeSession.Run(runOptions, input_names, inputs, output_names, outputs);
|
||||
|
||||
return lastHiddenState;
|
||||
}
|
||||
```
|
||||
|
||||
```text
|
||||
torch.Size([1, 77, 768])
|
||||
tensor([[[-0.3884, 0.0229, -0.0522, ..., -0.4899, -0.3066, 0.0675],
|
||||
|
|
@ -257,27 +284,37 @@ var latents = GenerateLatentSample(batchSize, height, width,seed, scheduler.Init
|
|||
For each inference step the latent image is duplicated to create the tensor shape of (2,4,64,64), it is then scaled and inferenced with the unet model. The output tensors (2,4,64,64) are split and guidance is applied. The resulting tensor is then sent into the `LMSDiscreteScheduler` step as part of the denoising process and the resulting tensor from the scheduler step is returned and the loop completes again until the `num_inference_steps` is reached.
|
||||
|
||||
```csharp
|
||||
var modelPath = Directory.GetCurrentDirectory().ToString() + ("\\unet\\model.onnx");
|
||||
var scheduler = new LMSDiscreteScheduler();
|
||||
var timesteps = scheduler.SetTimesteps(numInferenceSteps);
|
||||
|
||||
var seed = new Random().Next();
|
||||
var latents = GenerateLatentSample(batchSize, height, width, seed, scheduler.InitNoiseSigma);
|
||||
|
||||
// Create Inference Session
|
||||
var unetSession = new InferenceSession(modelPath, options);
|
||||
var input = new List<NamedOnnxValue>();
|
||||
using var options = new SessionOptions();
|
||||
using var unetSession = new InferenceSession(modelPath, options);
|
||||
|
||||
var latentInputShape = new int[] { 2, 4, height / 8, width / 8 };
|
||||
var splitTensorsShape = new int[] { 1, 4, height / 8, width / 8 };
|
||||
|
||||
for (int t = 0; t < timesteps.Length; t++)
|
||||
{
|
||||
// torch.cat([latents] * 2)
|
||||
var latentModelInput = TensorHelper.Duplicate(latents.ToArray(), new[] { 2, 4, height / 8, width / 8 });
|
||||
|
||||
var latentModelInput = TensorHelper.Duplicate(latents.ToArray(), latentInputShape);
|
||||
|
||||
// Scale the input
|
||||
latentModelInput = scheduler.ScaleInput(latentModelInput, timesteps[t]);
|
||||
|
||||
|
||||
// Create model input of text embeddings, scaled latent image and timestep
|
||||
input = CreateUnetModelInput(textEmbeddings, latentModelInput, timesteps[t]);
|
||||
|
||||
var input = CreateUnetModelInput(textEmbeddings, latentModelInput, timesteps[t]);
|
||||
|
||||
// Run Inference
|
||||
var output = unetSession.Run(input);
|
||||
var outputTensor = (output.ToList().First().Value as DenseTensor<float>);
|
||||
using var output = unetSession.Run(input);
|
||||
var outputTensor = output[0].Value as DenseTensor<float>;
|
||||
|
||||
// Split tensors from 2,4,64,64 to 1,4,64,64
|
||||
var splitTensors = TensorHelper.SplitTensor(outputTensor, new[] { 1, 4, height / 8, width / 8 });
|
||||
var splitTensors = TensorHelper.SplitTensor(outputTensor, splitTensorsShape);
|
||||
var noisePred = splitTensors.Item1;
|
||||
var noisePredText = splitTensors.Item2;
|
||||
|
||||
|
|
@ -287,6 +324,7 @@ for (int t = 0; t < timesteps.Length; t++)
|
|||
// LMS Scheduler Step
|
||||
latents = scheduler.Step(noisePred, timesteps[t], latents);
|
||||
}
|
||||
|
||||
```
|
||||
## Postprocess the `output` with the VAEDecoder
|
||||
After the inference loop is complete, the resulting tensor is scaled and then sent to the `vae_decoder` model to decode the image. Lastly the decoded image tensor is converted to an image and saved to disc.
|
||||
|
|
|
|||
Loading…
Reference in a new issue