diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/LabelMap.cs b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/LabelMap.cs new file mode 100644 index 0000000000..3dccd927b7 --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/LabelMap.cs @@ -0,0 +1,87 @@ +namespace Microsoft.ML.OnnxRuntime.FasterRcnnSample +{ + public class LabelMap + { + public static readonly string[] Labels = new[] {"__background", + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush"}; + } +} \ No newline at end of file diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Microsoft.ML.OnnxRuntime.FasterRcnnSample.csproj b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Microsoft.ML.OnnxRuntime.FasterRcnnSample.csproj new file mode 100644 index 0000000000..757ca6ab5a --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Microsoft.ML.OnnxRuntime.FasterRcnnSample.csproj @@ -0,0 +1,15 @@ + + + + Exe + netcoreapp3.1 + 8.0 + + + + + + + + + \ No newline at end of file diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Prediction.cs b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Prediction.cs new file mode 100644 index 0000000000..4857f2d83f --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Prediction.cs @@ -0,0 +1,26 @@ +namespace Microsoft.ML.OnnxRuntime.FasterRcnnSample +{ + public class Prediction + { + public Box Box { get; set; } + public string Label { get; set; } + public float Confidence { get; set; } + } + + public class Box + { + public float Xmin { get; set; } + public float Ymin { get; set; } + public float Xmax { get; set; } + public float Ymax { get; set; } + + public Box(float xmin, float ymin, float xmax, float ymax) + { + Xmin = xmin; + Ymin = ymin; + Xmax = xmax; + Ymax = ymax; + + } + } +} \ No newline at end of file diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Program.cs b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Program.cs new file mode 100644 index 0000000000..88edca5c77 --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/Program.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.ML.OnnxRuntime.Tensors; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Formats; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; +using SixLabors.ImageSharp.Drawing.Processing; +using SixLabors.Fonts; + +namespace Microsoft.ML.OnnxRuntime.FasterRcnnSample +{ + class Program + { + public static void Main(string[] args) + { + // Read paths + string modelFilePath = args[0]; + string imageFilePath = args[1]; + string outImageFilePath = args[2]; + + // Read image + using Image image = Image.Load(imageFilePath, out IImageFormat format); + + // Resize image + float ratio = 800f / Math.Min(image.Width, image.Height); + using Stream imageStream = new MemoryStream(); + image.Mutate(x => x.Resize((int)(ratio * image.Width), (int)(ratio * image.Height))); + image.Save(imageStream, format); + + // Preprocess image + var paddedHeight = (int)(Math.Ceiling(image.Height / 32f) * 32f); + var paddedWidth = (int)(Math.Ceiling(image.Width / 32f) * 32f); + Tensor input = new DenseTensor(new[] { 3, paddedHeight, paddedWidth }); + var mean = new[] { 102.9801f, 115.9465f, 122.7717f }; + for (int y = paddedHeight - image.Height; y < image.Height; y++) + { + Span pixelSpan = image.GetPixelRowSpan(y); + for (int x = paddedWidth - image.Width; x < image.Width; x++) + { + input[0, y, x] = pixelSpan[x].B - mean[0]; + input[1, y, x] = pixelSpan[x].G - mean[1]; + input[2, y, x] = pixelSpan[x].R - mean[2]; + } + } + + // Setup inputs and outputs + var inputs = new List + { + NamedOnnxValue.CreateFromTensor("image", input) + }; + + // Run inference + using var session = new InferenceSession(modelFilePath); + using IDisposableReadOnlyCollection results = session.Run(inputs); + + // Postprocess to get predictions + var resultsArray = results.ToArray(); + float[] boxes = resultsArray[0].AsEnumerable().ToArray(); + long[] labels = resultsArray[1].AsEnumerable().ToArray(); + float[] confidences = resultsArray[2].AsEnumerable().ToArray(); + var predictions = new List(); + var minConfidence = 0.7f; + for (int i = 0; i < boxes.Length - 4; i += 4) + { + var index = i / 4; + if (confidences[index] >= minConfidence) + { + predictions.Add(new Prediction + { + Box = new Box(boxes[i], boxes[i + 1], boxes[i + 2], boxes[i + 3]), + Label = LabelMap.Labels[labels[index]], + Confidence = confidences[index] + }); + } + } + + // Put boxes, labels and confidence on image and save for viewing + using var outputImage = File.OpenWrite(outImageFilePath); + Font font = SystemFonts.CreateFont("Arial", 16); + foreach (var p in predictions) + { + image.Mutate(x => + { + x.DrawLines(Color.Red, 2f, new PointF[] { + + new PointF(p.Box.Xmin, p.Box.Ymin), + new PointF(p.Box.Xmax, p.Box.Ymin), + + new PointF(p.Box.Xmax, p.Box.Ymin), + new PointF(p.Box.Xmax, p.Box.Ymax), + + new PointF(p.Box.Xmax, p.Box.Ymax), + new PointF(p.Box.Xmin, p.Box.Ymax), + + new PointF(p.Box.Xmin, p.Box.Ymax), + new PointF(p.Box.Xmin, p.Box.Ymin) + }); + x.DrawText($"{p.Label}, {p.Confidence:0.00}", font, Color.White, new PointF(p.Box.Xmin, p.Box.Ymin)); + }); + } + image.Save(outputImage, format); + } + } +} \ No newline at end of file diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/README.md b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/README.md new file mode 100644 index 0000000000..5b042da0c6 --- /dev/null +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/README.md @@ -0,0 +1,172 @@ +# C# Sample: Faster R-CNN + +The sample walks through how to run a pretrained Faster R-CNN object detection ONNX model using the ONNX Runtime C# API. + +The source code for this sample is available [here](Program.cs). + +## Prerequisites + +To run this sample, you'll need the following things: + +1. Install [.NET Core 3.1](https://dotnet.microsoft.com/download/dotnet-core/3.1) or higher for you OS (Mac, Windows or Linux). +2. Download the [Faster R-CNN](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/faster-rcnn/model/FasterRCNN-10.onnx) ONNX model to your local system. +3. Download [this demo image](demo.jpg) to test the model. You can also use any image you like. + +## Getting Started + +Now we have everything set up, we can start adding code to run the model on the image. We'll do this in the main method of the program for simplicity. + +### Read paths + +Firstly, let's read the path to the model, path to the image we want to test, and path to the output image: + +```cs +string modelFilePath = args[0]; +string imageFilePath = args[1]; +string outImageFilePath = args[2]; +``` + +### Read image + +Next, we will read the image in using the cross-platform image library [ImageSharp](https://www.nuget.org/packages/SixLabors.ImageSharp): + +```cs +using Image image = Image.Load(imageFilePath, out IImageFormat format); +``` + +Note, we're specifically reading the `Rgb24` type so we can efficiently preprocess the image in a later step. + +### Resize image + +Next, we will resize the image to the appropriate size that the model is expecting; it is recommended to resize the image such that both height and width are within the range of [800, 1333]. + +```cs +float ratio = 800f / Math.Min(image.Width, image.Height); +using Stream imageStream = new MemoryStream(); +image.Mutate(x => x.Resize((int)(ratio * image.Width), (int)(ratio * image.Height))); +image.Save(imageStream, format); +``` + +### Preprocess image + +Next, we will preprocess the image according to the [requirements of the model](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/faster-rcnn#preprocessing-steps): + +```cs +var paddedHeight = (int)(Math.Ceiling(image.Height / 32f) * 32f); +var paddedWidth = (int)(Math.Ceiling(image.Width / 32f) * 32f); +Tensor input = new DenseTensor(new[] { 3, paddedHeight, paddedWidth }); +var mean = new[] { 102.9801f, 115.9465f, 122.7717f }; +for (int y = paddedHeight - image.Height; y < image.Height; y++) +{ + Span pixelSpan = image.GetPixelRowSpan(y); + for (int x = paddedWidth - image.Width; x < image.Width; x++) + { + input[0, y, x] = pixelSpan[x].B - mean[0]; + input[1, y, x] = pixelSpan[x].G - mean[1]; + input[2, y, x] = pixelSpan[x].R - mean[2]; + } +} +``` + +Here, we're creating a Tensor of the required size `(channels, paddedHeight, paddedWidth)`, accessing the pixel values, preprocessing them and finally assigning them to the tensor at the appropriate indicies. + +### Setup inputs + +Next, we will create the inputs to the model: + +```cs +var inputs = new List +{ + NamedOnnxValue.CreateFromTensor("image", input) +}; +``` + +To check the input node names for an ONNX model, you can use [Netron](https://github.com/lutzroeder/netron) to visualise the model and see input/output names. In this case, this model has `image` as the input node name. + +### Run inference + +Next, we will create an inference session and run the input through it: + +```cs +using var session = new InferenceSession(modelFilePath); +using IDisposableReadOnlyCollection results = session.Run(inputs); +``` + +### Postprocess output + +Next, we will need to postprocess the output to get boxes and associated label and confidence scores for each box: + +```cs +var resultsArray = results.ToArray(); +float[] boxes = resultsArray[0].AsEnumerable().ToArray(); +long[] labels = resultsArray[1].AsEnumerable().ToArray(); +float[] confidences = resultsArray[2].AsEnumerable().ToArray(); +var predictions = new List(); +var minConfidence = 0.7f; +for (int i = 0; i < boxes.Length - 4; i += 4) +{ + var index = i / 4; + if (confidences[index] >= minConfidence) + { + predictions.Add(new Prediction + { + Box = new Box(boxes[i], boxes[i + 1], boxes[i + 2], boxes[i + 3]), + Label = LabelMap.Labels[labels[index]], + Confidence = confidences[index] + }); + } +} +``` + +Note, we're only taking boxes that have a confidence above 0.7 to remove false positives. + +### View prediction + +Next, we'll draw the boxes and associated labels and confidence scores on the image to see how the model went: + +```cs +using var outputImage = File.OpenWrite(outImageFilePath); +Font font = SystemFonts.CreateFont("Arial", 16); +foreach (var p in predictions) +{ + image.Mutate(x => + { + x.DrawLines(Color.Red, 2f, new PointF[] { + + new PointF(p.Box.Xmin, p.Box.Ymin), + new PointF(p.Box.Xmax, p.Box.Ymin), + + new PointF(p.Box.Xmax, p.Box.Ymin), + new PointF(p.Box.Xmax, p.Box.Ymax), + + new PointF(p.Box.Xmax, p.Box.Ymax), + new PointF(p.Box.Xmin, p.Box.Ymax), + + new PointF(p.Box.Xmin, p.Box.Ymax), + new PointF(p.Box.Xmin, p.Box.Ymin) + }); + x.DrawText($"{p.Label}, {p.Confidence:0.00}", font, Color.White, new PointF(p.Box.Xmin, p.Box.Ymin)); + }); +} +image.Save(outputImage, format); +``` + +For each box prediction, we're using ImageSharp to draw red lines to create the boxes, and drawing the label and confidence text. + +## Running the program + +Now the program is created, we can run it will the following command: + +``` +dotnet run [path-to-model] [path-to-image] [path-to-output-image] +``` + +e.g. running: + +``` +dotnet run ~/Downloads/FasterRCNN-10.onnx ~/Downloads/demo.jpg ~/Downloads/out.jpg +``` + +detects the following objects in the image: + +![](out.jpg) \ No newline at end of file diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/demo.jpg b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/demo.jpg new file mode 100644 index 0000000000..3a54056b78 Binary files /dev/null and b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/demo.jpg differ diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/out.jpg b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/out.jpg new file mode 100644 index 0000000000..fa44e0135a Binary files /dev/null and b/csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample/out.jpg differ diff --git a/samples/README.md b/samples/README.md index 101c70a369..800c590289 100644 --- a/samples/README.md +++ b/samples/README.md @@ -39,6 +39,7 @@ For a list of available dockerfiles and published images to help with getting st ## C# * [Inference Tutorial](../docs/CSharp_API.md#getting-started) * [ResNet50 v2 Tutorial](../csharp/sample/Microsoft.ML.OnnxRuntime.ResNet50v2Sample) +* [Faster R-CNN Tutorial](../csharp/sample/Microsoft.ML.OnnxRuntime.FasterRcnnSample) ## C/C++ * [C: SqueezeNet](../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp)