From 3effac2990fe73a6725d27748ba52f19f7b70e8c Mon Sep 17 00:00:00 2001 From: Josh Bradley Date: Thu, 9 Jul 2020 02:17:50 -0400 Subject: [PATCH] Experimental C++ API examples (#4358) * Add examples * fix build instructions for linux users * fix header include * update documentation --- samples/README.md | 3 +- samples/c_cxx/CMakeLists.txt | 15 ++- samples/c_cxx/model-explorer/CMakeLists.txt | 8 ++ .../model-explorer/batch-model-explorer.cpp | 125 ++++++++++++++++++ .../c_cxx/model-explorer/model-explorer.cpp | 106 +++++++++++++++ 5 files changed, 249 insertions(+), 8 deletions(-) create mode 100644 samples/c_cxx/model-explorer/CMakeLists.txt create mode 100644 samples/c_cxx/model-explorer/batch-model-explorer.cpp create mode 100644 samples/c_cxx/model-explorer/model-explorer.cpp diff --git a/samples/README.md b/samples/README.md index b374f63ec2..4e7f1c5e6b 100644 --- a/samples/README.md +++ b/samples/README.md @@ -41,8 +41,9 @@ For a list of available dockerfiles and published images to help with getting st ## C/C++ * [C: SqueezeNet](../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp) +* [C++: model-explorer](./c_cxx/model-explorer) - single and batch processing * [C++: SqueezeNet](../csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp) -* [C++:MNIST)](./c_cxx/MNIST) +* [C++: MNIST](./c_cxx/MNIST) ## Java * [Inference Tutorial](../docs/Java_API.md#getting-started) diff --git a/samples/c_cxx/CMakeLists.txt b/samples/c_cxx/CMakeLists.txt index 18b5797eea..d4b37dcba6 100644 --- a/samples/c_cxx/CMakeLists.txt +++ b/samples/c_cxx/CMakeLists.txt @@ -32,10 +32,8 @@ if(NOT ONNXRUNTIME_ROOTDIR) endif() endif() -if(WIN32) - include_directories("${ONNXRUNTIME_ROOTDIR}/include" "${ONNXRUNTIME_ROOTDIR}/include/onnxruntime/core/session") - link_directories("${ONNXRUNTIME_ROOTDIR}/lib") -endif() +include_directories("${ONNXRUNTIME_ROOTDIR}/include" "${ONNXRUNTIME_ROOTDIR}/include/onnxruntime/core/session") +link_directories("${ONNXRUNTIME_ROOTDIR}/lib") #if JPEG lib is available, we'll use it for image decoding, otherwise we'll use WIC find_package(JPEG) @@ -78,9 +76,12 @@ if(onnxruntime_USE_DML) add_definitions(-DUSE_DML) endif() - -add_subdirectory(imagenet) +# some examples require a Windows build environment +if(WIN32) + add_subdirectory(imagenet) + add_subdirectory(MNIST) +endif() if(PNG_FOUND) add_subdirectory(fns_candy_style_transfer) endif() -add_subdirectory(MNIST) +add_subdirectory(model-explorer) diff --git a/samples/c_cxx/model-explorer/CMakeLists.txt b/samples/c_cxx/model-explorer/CMakeLists.txt new file mode 100644 index 0000000000..164dad4ed5 --- /dev/null +++ b/samples/c_cxx/model-explorer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +add_executable(model-explorer model-explorer.cpp) +target_link_libraries(model-explorer PRIVATE onnxruntime) + +add_executable(batch-model-explorer batch-model-explorer.cpp) +target_link_libraries(batch-model-explorer PRIVATE onnxruntime) \ No newline at end of file diff --git a/samples/c_cxx/model-explorer/batch-model-explorer.cpp b/samples/c_cxx/model-explorer/batch-model-explorer.cpp new file mode 100644 index 0000000000..8e5abcd882 --- /dev/null +++ b/samples/c_cxx/model-explorer/batch-model-explorer.cpp @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * This example demonstrates how to batch process data using the experimental C++ API. + * + * This example is based on the model-explorer.cpp example except it demonstrates how to + * batch process data. Please start by checking out model-explorer.cpp first. + * + * This example is best run with one of the ResNet models (i.e. ResNet18) from the onnx model zoo at + * https://github.com/onnx/models + * + * Assumptions made in this example: + * 1) The onnx model has 1 input node and 1 output node + * 2) The onnx model has a symbolic first dimension (i.e. -1x3x224x224) + * + * + * In this example, we do the following: + * 1) read in an onnx model + * 2) print out some metadata information about inputs and outputs that the model expects + * 3) create tensors by generating 3 random batches of data (with batch_size = 5) for input to the model + * 4) pass each batch through the model and check the resulting output + * + * + * NOTE: Some onnx models may not have a symbolic first dimension. To prepare the onnx model, see the python code snippet below. + * ============= Python Example ====================== + * import onnx + * model = onnx.load_model('model.onnx') + * model.graph.input[0].type.tensor_type.shape.dim[0].dim_param = 'None' + * onnx.save_model(model, 'model-symbolic.onnx') + * + */ + +#include // std::generate +#include +#include +#include +#include +#include + +// pretty prints a shape dimension vector +std::string print_shape(const std::vector& v) { + std::stringstream ss(""); + for (size_t i = 0; i < v.size() - 1; i++) + ss << v[i] << "x"; + ss << v[v.size() - 1]; + return ss.str(); +} + +int calculate_product(const std::vector& v) { + int total = 1; + for (auto& i : v) total *= i; + return total; +} + +using namespace std; + +int main(int argc, char** argv) { + if (argc != 2) { + cout << "Usage: ./onnx-api-example " << endl; + return -1; + } + // onnxruntime setup + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "batch-model-explorer"); + Ort::SessionOptions session_options; + Ort::Experimental::Session session = Ort::Experimental::Session(env, argv[1], session_options); + + // print name/shape of inputs + auto input_names = session.GetInputNames(); + auto input_shapes = session.GetInputShapes(); + cout << "Input Node Name/Shape (" << input_names.size() << "):" << endl; + for (size_t i = 0; i < input_names.size(); i++) { + cout << "\t" << input_names[i] << " : " << print_shape(input_shapes[i]) << endl; + } + + // print name/shape of outputs + auto output_names = session.GetOutputNames(); + auto output_shapes = session.GetOutputShapes(); + cout << "Output Node Name/Shape (" << output_names.size() << "):" << endl; + for (size_t i = 0; i < output_names.size(); i++) { + cout << "\t" << output_names[i] << " : " << print_shape(output_shapes[i]) << endl; + } + + // Assume model has 1 input node and 1 output node. + assert(input_names.size() == 1 && output_names.size() == 1); + + int batch_size = 5; + int num_batches = 3; + auto input_shape = input_shapes[0]; + assert(input_shape[0] == -1); // symbolic dimensions are represented by a -1 value + input_shape[0] = batch_size; + int num_elements_per_batch = calculate_product(input_shape); + + // process multiple batches + for (int i = 0; i < num_batches; i++) { + cout << "\nProcessing batch #" << i << endl; + + // Create an Ort tensor containing random numbers + std::vector batch_input_tensor_values(num_elements_per_batch); + std::generate(batch_input_tensor_values.begin(), batch_input_tensor_values.end(), [&] { return rand() % 255; }); // generate random numbers in the range [0, 255] + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + std::vector batch_input_tensors; + batch_input_tensors.push_back(Ort::Value::CreateTensor(memory_info, batch_input_tensor_values.data(), batch_input_tensor_values.size(), input_shape.data(), input_shape.size())); + + // double-check the dimensions of the input tensor + assert(batch_input_tensors[0].IsTensor() && + batch_input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == input_shape); + cout << "batch_input_tensor shape: " << print_shape(batch_input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << endl; + + // pass data through model + try { + auto batch_output_tensors = session.Run(input_names, batch_input_tensors, output_names); + // double-check the dimensions of the output tensors + // NOTE: the number of output tensors is equal to the number of output nodes specifed in the Run() call + assert(batch_output_tensors.size() == output_names.size() && + batch_output_tensors[0].IsTensor() && + batch_output_tensors[0].GetTensorTypeAndShapeInfo().GetShape()[0] == batch_size); + cout << "batch_output_tensor_shape: " << print_shape(batch_output_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << endl; + } catch (const Ort::Exception& exception) { + cout << "ERROR running model inference: " << exception.what() << endl; + exit(-1); + } + } + cout << "\nDone" << endl; +} diff --git a/samples/c_cxx/model-explorer/model-explorer.cpp b/samples/c_cxx/model-explorer/model-explorer.cpp new file mode 100644 index 0000000000..3d9565d134 --- /dev/null +++ b/samples/c_cxx/model-explorer/model-explorer.cpp @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * This sample application demonstrates how to use components of the experimental C++ API + * to query for model inputs/outputs and how to run inferrence on a model. + * + * This example is best run with one of the ResNet models (i.e. ResNet18) from the onnx model zoo at + * https://github.com/onnx/models + * + * Assumptions made in this example: + * 1) The onnx model has 1 input node and 1 output node + * + * + * In this example, we do the following: + * 1) read in an onnx model + * 2) print out some metadata information about inputs and outputs that the model expects + * 3) generate random data for an input tensor + * 4) pass tensor through the model and check the resulting tensor + * + */ + +#include // std::generate +#include +#include +#include +#include +#include + +// pretty prints a shape dimension vector +std::string print_shape(const std::vector& v) { + std::stringstream ss(""); + for (size_t i = 0; i < v.size() - 1; i++) + ss << v[i] << "x"; + ss << v[v.size() - 1]; + return ss.str(); +} + +int calculate_product(const std::vector& v) { + int total = 1; + for (auto& i : v) total *= i; + return total; +} + +using namespace std; + +int main(int argc, char** argv) { + if (argc != 2) { + cout << "Usage: ./onnx-api-example " << endl; + return -1; + } + // onnxruntime setup + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "example-model-explorer"); + Ort::SessionOptions session_options; + Ort::Experimental::Session session = Ort::Experimental::Session(env, argv[1], session_options); // access experimental components via the Experimental namespace + + // print name/shape of inputs + std::vector input_names = session.GetInputNames(); + std::vector > input_shapes = session.GetInputShapes(); + cout << "Input Node Name/Shape (" << input_names.size() << "):" << endl; + for (size_t i = 0; i < input_names.size(); i++) { + cout << "\t" << input_names[i] << " : " << print_shape(input_shapes[i]) << endl; + } + + // print name/shape of outputs + std::vector output_names = session.GetOutputNames(); + std::vector > output_shapes = session.GetOutputShapes(); + cout << "Output Node Name/Shape (" << output_names.size() << "):" << endl; + for (size_t i = 0; i < output_names.size(); i++) { + cout << "\t" << output_names[i] << " : " << print_shape(output_shapes[i]) << endl; + } + + // Assume model has 1 input node and 1 output node. + assert(input_names.size() == 1 && output_names.size() == 1); + + // Create a single Ort tensor of random numbers + auto input_shape = input_shapes[0]; + int total_number_elements = calculate_product(input_shape); + std::vector input_tensor_values(total_number_elements); + std::generate(input_tensor_values.begin(), input_tensor_values.end(), [&] { return rand() % 255; }); // generate random numbers in the range [0, 255] + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + std::vector input_tensors; + input_tensors.push_back(Ort::Value::CreateTensor(memory_info, input_tensor_values.data(), input_tensor_values.size(), input_shape.data(), input_shape.size())); + + // double-check the dimensions of the input tensor + assert(input_tensors[0].IsTensor() && + input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == input_shape); + cout << "\ninput_tensor shape: " << print_shape(input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << endl; + + // pass data through model + cout << "Running model..."; + try { + auto output_tensors = session.Run(session.GetInputNames(), input_tensors, session.GetOutputNames()); + cout << "done" << endl; + + // double-check the dimensions of the output tensors + // NOTE: the number of output tensors is equal to the number of output nodes specifed in the Run() call + assert(output_tensors.size() == session.GetOutputNames().size() && + output_tensors[0].IsTensor()); + cout << "output_tensor_shape: " << print_shape(output_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << endl; + + } catch (const Ort::Exception& exception) { + cout << "ERROR running model inference: " << exception.what() << endl; + exit(-1); + } +}