saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

C# samples: Faster R-CNN (#4733)

Browse source 
* C# sample: Faster R-CNN

* Add link to new sample in samples README

* Remove duplicate image
This commit is contained in:
Marcus Turewicz 2020-08-14 10:05:01 +10:00 committed by GitHub
parent de2685261b
commit ce65275edf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 408 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/LabelMap.cs Normal file
View file

@ -0,0 +1,87 @@
namespace Microsoft.ML.OnnxRuntime.FasterRcnnSample
{
public class LabelMap
{
public static readonly string[] Labels = new[] {"__background",
"person",
"bicycle",
"car",
"motorcycle",
"airplane",
"bus",
"train",
"truck",
"boat",
"traffic light",
"fire hydrant",
"stop sign",
"parking meter",
"bench",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"backpack",
"umbrella",
"handbag",
"tie",
"suitcase",
"frisbee",
"skis",
"snowboard",
"sports ball",
"kite",
"baseball bat",
"baseball glove",
"skateboard",
"surfboard",
"tennis racket",
"bottle",
"wine glass",
"cup",
"fork",
"knife",
"spoon",
"bowl",
"banana",
"apple",
"sandwich",
"orange",
"broccoli",
"carrot",
"hot dog",
"pizza",
"donut",
"cake",
"chair",
"couch",
"potted plant",
"bed",
"dining table",
"toilet",
"tv",
"laptop",
"mouse",
"remote",
"keyboard",
"cell phone",
"microwave",
"oven",
"toaster",
"sink",
"refrigerator",
"book",
"clock",
"vase",
"scissors",
"teddy bear",
"hair drier",
"toothbrush"};
}
}

15
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Microsoft.ML.OnnxRuntime.FasterRcnnSample.csproj Normal file
View file

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<LangVersion>8.0</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.4.0" />
<PackageReference Include="Sixlabors.ImageSharp" Version="1.0.0" />
<PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta0010" />
</ItemGroup>
</Project>

26
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Prediction.cs Normal file
View file

@ -0,0 +1,26 @@
namespace Microsoft.ML.OnnxRuntime.FasterRcnnSample
{
public class Prediction
{
public Box Box { get; set; }
public string Label { get; set; }
public float Confidence { get; set; }
}
public class Box
{
public float Xmin { get; set; }
public float Ymin { get; set; }
public float Xmax { get; set; }
public float Ymax { get; set; }
public Box(float xmin, float ymin, float xmax, float ymax)
{
Xmin = xmin;
Ymin = ymin;
Xmax = xmax;
Ymax = ymax;
}
}
}

107
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Program.cs Normal file
View file

@ -0,0 +1,107 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.ML.OnnxRuntime.Tensors;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using SixLabors.ImageSharp.Drawing.Processing;
using SixLabors.Fonts;
namespace Microsoft.ML.OnnxRuntime.FasterRcnnSample
{
class Program
{
public static void Main(string[] args)
{
// Read paths
string modelFilePath = args[0];
string imageFilePath = args[1];
string outImageFilePath = args[2];
// Read image
using Image<Rgb24> image = Image.Load<Rgb24>(imageFilePath, out IImageFormat format);
// Resize image
float ratio = 800f / Math.Min(image.Width, image.Height);
using Stream imageStream = new MemoryStream();
image.Mutate(x => x.Resize((int)(ratio * image.Width), (int)(ratio * image.Height)));
image.Save(imageStream, format);
// Preprocess image
var paddedHeight = (int)(Math.Ceiling(image.Height / 32f) * 32f);
var paddedWidth = (int)(Math.Ceiling(image.Width / 32f) * 32f);
Tensor<float> input = new DenseTensor<float>(new[] { 3, paddedHeight, paddedWidth });
var mean = new[] { 102.9801f, 115.9465f, 122.7717f };
for (int y = paddedHeight - image.Height; y < image.Height; y++)
{
Span<Rgb24> pixelSpan = image.GetPixelRowSpan(y);
for (int x = paddedWidth - image.Width; x < image.Width; x++)
{
input[0, y, x] = pixelSpan[x].B - mean[0];
input[1, y, x] = pixelSpan[x].G - mean[1];
input[2, y, x] = pixelSpan[x].R - mean[2];
}
}
// Setup inputs and outputs
var inputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("image", input)
};
// Run inference
using var session = new InferenceSession(modelFilePath);
using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.Run(inputs);
// Postprocess to get predictions
var resultsArray = results.ToArray();
float[] boxes = resultsArray[0].AsEnumerable<float>().ToArray();
long[] labels = resultsArray[1].AsEnumerable<long>().ToArray();
float[] confidences = resultsArray[2].AsEnumerable<float>().ToArray();
var predictions = new List<Prediction>();
var minConfidence = 0.7f;
for (int i = 0; i < boxes.Length - 4; i += 4)
{
var index = i / 4;
if (confidences[index] >= minConfidence)
{
predictions.Add(new Prediction
{
Box = new Box(boxes[i], boxes[i + 1], boxes[i + 2], boxes[i + 3]),
Label = LabelMap.Labels[labels[index]],
Confidence = confidences[index]
});
}
}
// Put boxes, labels and confidence on image and save for viewing
using var outputImage = File.OpenWrite(outImageFilePath);
Font font = SystemFonts.CreateFont("Arial", 16);
foreach (var p in predictions)
{
image.Mutate(x =>
{
x.DrawLines(Color.Red, 2f, new PointF[] {
new PointF(p.Box.Xmin, p.Box.Ymin),
new PointF(p.Box.Xmax, p.Box.Ymin),
new PointF(p.Box.Xmax, p.Box.Ymin),
new PointF(p.Box.Xmax, p.Box.Ymax),
new PointF(p.Box.Xmax, p.Box.Ymax),
new PointF(p.Box.Xmin, p.Box.Ymax),
new PointF(p.Box.Xmin, p.Box.Ymax),
new PointF(p.Box.Xmin, p.Box.Ymin)
});
x.DrawText($"{p.Label}, {p.Confidence:0.00}", font, Color.White, new PointF(p.Box.Xmin, p.Box.Ymin));
});
}
image.Save(outputImage, format);
}
}
}

172
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/README.md Normal file
View file

@ -0,0 +1,172 @@
# C# Sample: Faster R-CNN
The sample walks through how to run a pretrained Faster R-CNN object detection ONNX model using the ONNX Runtime C# API.
The source code for this sample is available [here](Program.cs).
## Prerequisites
To run this sample, you'll need the following things:
1. Install [.NET Core 3.1](https://dotnet.microsoft.com/download/dotnet-core/3.1) or higher for you OS (Mac, Windows or Linux).
2. Download the [Faster R-CNN](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/faster-rcnn/model/FasterRCNN-10.onnx) ONNX model to your local system.
3. Download [this demo image](demo.jpg) to test the model. You can also use any image you like.
## Getting Started
Now we have everything set up, we can start adding code to run the model on the image. We'll do this in the main method of the program for simplicity.
### Read paths
Firstly, let's read the path to the model, path to the image we want to test, and path to the output image:
```cs
string modelFilePath = args[0];
string imageFilePath = args[1];
string outImageFilePath = args[2];
```
### Read image
Next, we will read the image in using the cross-platform image library [ImageSharp](https://www.nuget.org/packages/SixLabors.ImageSharp):
```cs
using Image<Rgb24> image = Image.Load<Rgb24>(imageFilePath, out IImageFormat format);
```
Note, we're specifically reading the `Rgb24` type so we can efficiently preprocess the image in a later step.
### Resize image
Next, we will resize the image to the appropriate size that the model is expecting; it is recommended to resize the image such that both height and width are within the range of [800, 1333].
```cs
float ratio = 800f / Math.Min(image.Width, image.Height);
using Stream imageStream = new MemoryStream();
image.Mutate(x => x.Resize((int)(ratio * image.Width), (int)(ratio * image.Height)));
image.Save(imageStream, format);
```
### Preprocess image
Next, we will preprocess the image according to the [requirements of the model](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/faster-rcnn#preprocessing-steps):
```cs
var paddedHeight = (int)(Math.Ceiling(image.Height / 32f) * 32f);
var paddedWidth = (int)(Math.Ceiling(image.Width / 32f) * 32f);
Tensor<float> input = new DenseTensor<float>(new[] { 3, paddedHeight, paddedWidth });
var mean = new[] { 102.9801f, 115.9465f, 122.7717f };
for (int y = paddedHeight - image.Height; y < image.Height; y++)
{
Span<Rgb24> pixelSpan = image.GetPixelRowSpan(y);
for (int x = paddedWidth - image.Width; x < image.Width; x++)
{
input[0, y, x] = pixelSpan[x].B - mean[0];
input[1, y, x] = pixelSpan[x].G - mean[1];
input[2, y, x] = pixelSpan[x].R - mean[2];
}
}
```
Here, we're creating a Tensor of the required size `(channels, paddedHeight, paddedWidth)`, accessing the pixel values, preprocessing them and finally assigning them to the tensor at the appropriate indicies.
### Setup inputs
Next, we will create the inputs to the model:
```cs
var inputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("image", input)
};
```
To check the input node names for an ONNX model, you can use [Netron](https://github.com/lutzroeder/netron) to visualise the model and see input/output names. In this case, this model has `image` as the input node name.
### Run inference
Next, we will create an inference session and run the input through it:
```cs
using var session = new InferenceSession(modelFilePath);
using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.Run(inputs);
```
### Postprocess output
Next, we will need to postprocess the output to get boxes and associated label and confidence scores for each box:
```cs
var resultsArray = results.ToArray();
float[] boxes = resultsArray[0].AsEnumerable<float>().ToArray();
long[] labels = resultsArray[1].AsEnumerable<long>().ToArray();
float[] confidences = resultsArray[2].AsEnumerable<float>().ToArray();
var predictions = new List<Prediction>();
var minConfidence = 0.7f;
for (int i = 0; i < boxes.Length - 4; i += 4)
{
var index = i / 4;
if (confidences[index] >= minConfidence)
{
predictions.Add(new Prediction
{
Box = new Box(boxes[i], boxes[i + 1], boxes[i + 2], boxes[i + 3]),
Label = LabelMap.Labels[labels[index]],
Confidence = confidences[index]
});
}
}
```
Note, we're only taking boxes that have a confidence above 0.7 to remove false positives.
### View prediction
Next, we'll draw the boxes and associated labels and confidence scores on the image to see how the model went:
```cs
using var outputImage = File.OpenWrite(outImageFilePath);
Font font = SystemFonts.CreateFont("Arial", 16);
foreach (var p in predictions)
{
image.Mutate(x =>
{
x.DrawLines(Color.Red, 2f, new PointF[] {
new PointF(p.Box.Xmin, p.Box.Ymin),
new PointF(p.Box.Xmax, p.Box.Ymin),
new PointF(p.Box.Xmax, p.Box.Ymin),
new PointF(p.Box.Xmax, p.Box.Ymax),
new PointF(p.Box.Xmax, p.Box.Ymax),
new PointF(p.Box.Xmin, p.Box.Ymax),
new PointF(p.Box.Xmin, p.Box.Ymax),
new PointF(p.Box.Xmin, p.Box.Ymin)
});
x.DrawText($"{p.Label}, {p.Confidence:0.00}", font, Color.White, new PointF(p.Box.Xmin, p.Box.Ymin));
});
}
image.Save(outputImage, format);
```
For each box prediction, we're using ImageSharp to draw red lines to create the boxes, and drawing the label and confidence text.
## Running the program
Now the program is created, we can run it will the following command:
```
dotnet run [path-to-model] [path-to-image] [path-to-output-image]
```
e.g. running:
```
dotnet run ~/Downloads/FasterRCNN-10.onnx ~/Downloads/demo.jpg ~/Downloads/out.jpg
```
detects the following objects in the image:
![](out.jpg)

BIN
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/demo.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 167 KiB

BIN
csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/out.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 308 KiB

1
samples/README.md
View file

@ -39,6 +39,7 @@ For a list of available dockerfiles and published images to help with getting st
## C#
* [Inference Tutorial](../docs/CSharp_API.md#getting-started)
* [ResNet50 v2 Tutorial](../csharp/sample/Microsoft.ML.OnnxRuntime.ResNet50v2Sample)
* [Faster R-CNN Tutorial](../csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample)
## C/C++
* [C: SqueezeNet](../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp)