saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

C++ wrapper for ABI (#958)

Browse source
This commit is contained in:
Ryan Hill 2019-05-03 19:32:46 -07:00 committed by GitHub
parent 7329e99d52
commit f73ce305e9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 634 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp Normal file
View file

@ -0,0 +1,124 @@
// Copyright(c) Microsoft Corporation.All rights reserved.
// Licensed under the MIT License.
//
#include <assert.h>
#include <vector>
#include <onnxruntime_cxx_api.h>
int main(int argc, char* argv[]) {
//*************************************************************************
// initialize enviroment...one enviroment per process
// enviroment maintains thread pools and other state info
Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test");
// initialize session options if needed
Ort::SessionOptions session_options;
session_options.SetThreadPoolSize(1);
// If onnxruntime.dll is built with CUDA enabled, we can uncomment out this line to use CUDA for this
// session (we also need to include cuda_provider_factory.h above which defines it)
// #include "cuda_provider_factory.h"
// OrtSessionOptionsAppendExecutionProvider_CUDA(session_opsions, 1);
// Sets graph optimization level
// Available levels are
// 0 -> To disable all optimizations
// 1 -> To enable basic optimizations (Such as redundant node removals)
// 2 -> To enable all optimizations (Includes level 1 + more complex optimizations like node fusions)
session_options.SetGraphOptimizationLevel(1);
//*************************************************************************
// create session and load model into memory
// using squeezenet version 1.3
// URL = https://github.com/onnx/models/tree/master/squeezenet
const wchar_t* model_path = L"squeezenet.onnx";
Ort::Session session(env, model_path, session_options);
//*************************************************************************
// print model input layer (node names, types, shape etc.)
Ort::Allocator allocator = Ort::Allocator::Create_Default();
// print number of model input nodes
size_t num_input_nodes = session.GetInputCount();
std::vector<const char*> input_node_names(num_input_nodes);
std::vector<int64_t> input_node_dims; // simplify... this model has only 1 input node {1, 3, 224, 224}.
// Otherwise need vector<vector<>>
printf("Number of inputs = %zu\n", num_input_nodes);
// iterate over all input nodes
for (int i = 0; i < num_input_nodes; i++) {
// print input node names
char* input_name = session.GetInputName(i, allocator);
printf("Input %d : name=%s\n", i, input_name);
input_node_names[i] = input_name;
// print input node types
Ort::TypeInfo type_info = session.GetInputTypeInfo(i);
auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
ONNXTensorElementDataType type = tensor_info.GetElementType();
printf("Input %d : type=%d\n", i, type);
// print input shapes/dims
input_node_dims = tensor_info.GetShape();
printf("Input %d : num_dims=%zu\n", i, input_node_dims.size());
for (int j = 0; j < input_node_dims.size(); j++)
printf("Input %d : dim %d=%jd\n", i, j, input_node_dims[j]);
}
// Results should be...
// Number of inputs = 1
// Input 0 : name = data_0
// Input 0 : type = 1
// Input 0 : num_dims = 4
// Input 0 : dim 0 = 1
// Input 0 : dim 1 = 3
// Input 0 : dim 2 = 224
// Input 0 : dim 3 = 224
//*************************************************************************
// Similar operations to get output node information.
// Use OrtSessionGetOutputCount(), OrtSessionGetOutputName()
// OrtSessionGetOutputTypeInfo() as shown above.
//*************************************************************************
// Score the model using sample data, and inspect values
size_t input_tensor_size = 224 * 224 * 3; // simplify ... using known dim values to calculate size
// use OrtGetTensorShapeElementCount() to get official size!
std::vector<float> input_tensor_values(input_tensor_size);
std::vector<const char*> output_node_names = {"softmaxout_1"};
// initialize input data with values in [0.0, 1.0]
for (unsigned int i = 0; i < input_tensor_size; i++)
input_tensor_values[i] = (float)i / (input_tensor_size + 1);
// create input tensor object from data values
Ort::AllocatorInfo allocator_info = Ort::AllocatorInfo::Create_Cpu(OrtArenaAllocator, OrtMemTypeDefault);
Ort::Value input_tensors[1] = {Ort::Value::CreateTensor(allocator_info, input_tensor_values.data(), input_tensor_size * sizeof(float), input_node_dims.data(), 4, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)};
assert(input_tensors[0].IsTensor());
// score model & input tensor, get back output tensor
Ort::Value output_tensor = session.Run(nullptr, input_node_names.data(), input_tensors, output_node_names.data(), 1);
assert(output_tensor.IsTensor());
// Get pointer to output tensor float values
float* floatarr = output_tensor.GetTensorMutableData<float>();
assert(abs(floatarr[0] - 0.000045) < 1e-6);
// score the model, and print scores for first 5 classes
for (int i = 0; i < 5; i++)
printf("Score for class [%d] = %f\n", i, floatarr[i]);
// Results should be as below...
// Score for class[0] = 0.000045
// Score for class[1] = 0.003846
// Score for class[2] = 0.000125
// Score for class[3] = 0.001180
// Score for class[4] = 0.001317
printf("Done!\n");
return 0;
}

5
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -237,6 +237,7 @@ ORT_API(void, OrtSetSessionLogVerbosityLevel, _In_ OrtSessionOptions* options, u
ORT_API(int, OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions* options, uint32_t graph_optimization_level);
// How many threads in the session thread pool.
// Returns 0 on success, and -1 otherwise
ORT_API(int, OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int session_thread_pool_size);
/**
@ -553,11 +554,11 @@ struct OrtCustomOpApi {
size_t(ORT_API_CALL* GetDimensionCount)(_In_ const OrtTensorTypeAndShapeInfo* info);
void(ORT_API_CALL* GetDimensions)(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length);
OrtStatus*(ORT_API_CALL* SetDimensions)(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);
OrtStatus*(ORT_API_CALL* GetTensorMutableData)(_Inout_ OrtValue* value, void** data);
OrtStatus*(ORT_API_CALL* GetTensorMutableData)(_Inout_ OrtValue* value, _Out_ void** data);
void(ORT_API_CALL* ReleaseTensorTypeAndShapeInfo)(OrtTensorTypeAndShapeInfo* input);
OrtValue*(ORT_API_CALL* KernelContext_GetInput)(OrtKernelContext* context, _In_ size_t index);
const OrtValue*(ORT_API_CALL* KernelContext_GetInput)(const OrtKernelContext* context, _In_ size_t index);
OrtValue*(ORT_API_CALL* KernelContext_GetOutput)(OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count);
};
typedef struct OrtCustomOpApi OrtCustomOpApi;

513
include/onnxruntime/core/session/onnxruntime_cxx_api.h
View file

@ -3,79 +3,443 @@
#pragma once
#include "onnxruntime_c_api.h"
#include <vector>
#include <string>
#include <stdexcept>
#include <cstddef>
#include <array>
#include <algorithm>
#include <memory>
#include "core/common/exceptions.h"
//TODO: encode error code in the message?
#define ORT_THROW_ON_ERROR(expr) \
do { \
OrtStatus* onnx_status = (expr); \
if (onnx_status != nullptr) { \
std::string ort_error_message = OrtGetErrorMessage(onnx_status); \
OrtErrorCode error_code = OrtGetErrorCode(onnx_status); \
OrtReleaseStatus(onnx_status); \
switch (error_code) { \
case ORT_NOT_IMPLEMENTED: \
throw onnxruntime::NotImplementedException(ort_error_message); \
default: \
throw onnxruntime::OnnxRuntimeException(ORT_WHERE, ort_error_message); \
} \
} \
} while (0);
#include <stdexcept>
#include <string>
#include <vector>
#define ORT_REDIRECT_SIMPLE_FUNCTION_CALL(NAME) \
decltype(Ort##NAME(value.get())) NAME() { \
return Ort##NAME(value.get()); \
}
#define ORT_DEFINE_DELETER(NAME) \
template <> \
struct default_delete<Ort##NAME> { \
void operator()(Ort##NAME* ptr) { \
OrtRelease##NAME(ptr); \
} \
};
namespace std {
template <>
struct default_delete<OrtAllocator> {
void operator()(OrtAllocator* ptr) {
OrtReleaseAllocator(ptr);
}
};
template <>
struct default_delete<OrtEnv> {
void operator()(OrtEnv* ptr) {
OrtReleaseEnv(ptr);
}
};
template <>
struct default_delete<OrtRunOptions> {
void operator()(OrtRunOptions* ptr) {
OrtReleaseRunOptions(ptr);
}
};
template <>
struct default_delete<OrtTypeInfo> {
void operator()(OrtTypeInfo* ptr) {
OrtReleaseTypeInfo(ptr);
}
};
template <>
struct default_delete<OrtTensorTypeAndShapeInfo> {
void operator()(OrtTensorTypeAndShapeInfo* ptr) {
OrtReleaseTensorTypeAndShapeInfo(ptr);
}
};
template <>
struct default_delete<OrtSessionOptions> {
void operator()(OrtSessionOptions* ptr) {
OrtReleaseSessionOptions(ptr);
}
};
ORT_DEFINE_DELETER(Allocator);
ORT_DEFINE_DELETER(TypeInfo);
ORT_DEFINE_DELETER(RunOptions);
ORT_DEFINE_DELETER(SessionOptions);
ORT_DEFINE_DELETER(TensorTypeAndShapeInfo);
} // namespace std
namespace Ort {
using std::nullptr_t;
struct Exception : std::exception {
Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}
OrtErrorCode GetOrtErrorCode() const { return code_; }
const char* what() const noexcept { return message_.c_str(); }
private:
std::string message_;
OrtErrorCode code_;
};
#define ORT_THROW_ON_ERROR(expr) \
if (OrtStatus* onnx_status = (expr)) { \
std::string ort_error_message = OrtGetErrorMessage(onnx_status); \
OrtErrorCode ort_error_code = OrtGetErrorCode(onnx_status); \
OrtReleaseStatus(onnx_status); \
throw Ort::Exception(std::move(ort_error_message), ort_error_code); \
}
#define ORT_DEFINE_RELEASE(NAME) \
inline void Release(Ort##NAME* ptr) { OrtRelease##NAME(ptr); }
ORT_DEFINE_RELEASE(Allocator);
ORT_DEFINE_RELEASE(AllocatorInfo);
ORT_DEFINE_RELEASE(CustomOpDomain);
ORT_DEFINE_RELEASE(Env);
ORT_DEFINE_RELEASE(RunOptions);
ORT_DEFINE_RELEASE(Session);
ORT_DEFINE_RELEASE(SessionOptions);
ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);
ORT_DEFINE_RELEASE(TypeInfo);
ORT_DEFINE_RELEASE(Value);
template <typename T>
struct Base {
Base() = default;
Base(T* p) : p_{p} {}
~Base() { Release(p_); }
operator T*() { return p_; }
operator const T*() const { return p_; }
T* release() {
T* p = p_;
p_ = nullptr;
return p;
}
protected:
Base(const Base&) = delete;
Base(Base&& v) : p_{v.p_} { v.p_ = nullptr; }
void operator=(Base&& v) {
Release(p_);
p_ = v.p_;
v.p_ = nullptr;
}
T* p_{};
template <typename>
friend struct Unowned;
};
template <typename T>
struct Unowned : T {
Unowned(decltype(T::p_) p) : T{p} {}
Unowned(Unowned&& v) : T{v.p_} {}
~Unowned() { this->p_ = nullptr; }
};
struct TypeInfo;
struct Value;
struct Env : Base<OrtEnv> {
Env(nullptr_t) {}
Env(OrtLoggingLevel default_warning_level, _In_ const char* logid);
};
struct CustomOpDomain : Base<OrtCustomOpDomain> {
CustomOpDomain(nullptr_t) {}
CustomOpDomain(const char* domain);
void Add(OrtCustomOp* op);
};
struct RunOptions : Base<OrtRunOptions> {
RunOptions(nullptr_t) {}
RunOptions();
RunOptions& SetRunLogVerbosityLevel(unsigned int);
unsigned int GetRunLogVerbosityLevel() const;
RunOptions& SetRunTag(const char* run_tag);
const char* GetRunTag() const;
RunOptions& SetTerminate(bool flag);
};
struct SessionOptions : Base<OrtSessionOptions> {
SessionOptions(nullptr_t) {}
SessionOptions();
explicit SessionOptions(OrtSessionOptions* p) : Base<OrtSessionOptions>{p} {}
SessionOptions clone() const;
SessionOptions& SetThreadPoolSize(int session_thread_pool_size);
SessionOptions& SetGraphOptimizationLevel(uint32_t graph_optimization_level);
SessionOptions& EnableCpuMemArena();
SessionOptions& DisableCpuMemArena();
SessionOptions& EnableSequentialExecution();
SessionOptions& DisableSequentialExecution();
SessionOptions& SetLogId(const char* logid);
SessionOptions& Add(OrtCustomOpDomain* custom_op_domain);
};
struct Session : Base<OrtSession> {
Session(nullptr_t) {}
Session(OrtEnv* env, const ORTCHAR_T* model_path, const OrtSessionOptions* options);
template <unsigned InputCount>
Value Run(OrtRunOptions* run_options, const char* const* input_names, Value (&input)[InputCount],
const char* const* output_names, size_t output_names_len);
size_t GetInputCount() const;
size_t GetOutputCount() const;
char* GetInputName(size_t index, OrtAllocator* allocator) const;
char* GetOutputName(size_t index, OrtAllocator* allocator) const;
TypeInfo GetInputTypeInfo(size_t index) const;
TypeInfo GetOutputTypeInfo(size_t index) const;
};
struct TensorTypeAndShapeInfo : Base<OrtTensorTypeAndShapeInfo> {
TensorTypeAndShapeInfo(nullptr_t) {}
explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : Base<OrtTensorTypeAndShapeInfo>{p} {}
ONNXTensorElementDataType GetElementType() const;
size_t GetDimensionsCount() const;
void GetDimensions(int64_t* values, size_t values_count) const;
std::vector<int64_t> GetShape() const;
};
struct TypeInfo : Base<OrtTypeInfo> {
TypeInfo(nullptr_t) {}
explicit TypeInfo(OrtTypeInfo* p) : Base<OrtTypeInfo>{p} {}
Unowned<TensorTypeAndShapeInfo> GetTensorTypeAndShapeInfo() const;
};
struct Value : Base<OrtValue> {
static Value CreateTensor(const OrtAllocatorInfo* info, void* p_data, size_t p_data_len, const int64_t* shape, size_t shape_len,
ONNXTensorElementDataType type);
Value(nullptr_t) {}
explicit Value(OrtValue* p) : Base<OrtValue>{p} {}
bool IsTensor() const;
template <typename T>
T* GetTensorMutableData();
TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const;
};
struct Allocator : Base<OrtAllocator> {
static Allocator Create_Default();
Allocator(nullptr_t) {}
explicit Allocator(OrtAllocator* p) : Base<OrtAllocator>{p} {}
void* Alloc(size_t size);
void Free(void* p);
const OrtAllocatorInfo* GetInfo() const;
};
struct AllocatorInfo : Base<OrtAllocatorInfo> {
static AllocatorInfo Create_Cpu(OrtAllocatorType type, OrtMemType mem_type1);
AllocatorInfo(nullptr_t) {}
explicit AllocatorInfo(OrtAllocatorInfo* p) : Base<OrtAllocatorInfo>{p} {}
};
} // namespace Ort
namespace Ort {
inline Allocator Allocator::Create_Default() {
OrtAllocator* p;
ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&p));
return Allocator(p);
}
inline void* Allocator::Alloc(size_t size) {
return OrtAllocatorAlloc(p_, size);
}
inline void Allocator::Free(void* p) {
OrtAllocatorFree(p_, p);
}
inline const OrtAllocatorInfo* Allocator::GetInfo() const {
return OrtAllocatorGetInfo(p_);
}
inline AllocatorInfo AllocatorInfo::Create_Cpu(OrtAllocatorType type, OrtMemType mem_type) {
OrtAllocatorInfo* p;
ORT_THROW_ON_ERROR(OrtCreateCpuAllocatorInfo(type, mem_type, &p));
return AllocatorInfo(p);
}
inline Env::Env(OrtLoggingLevel default_warning_level, _In_ const char* logid) {
ORT_THROW_ON_ERROR(OrtCreateEnv(default_warning_level, logid, &p_));
}
inline CustomOpDomain::CustomOpDomain(const char* domain)
: Base<OrtCustomOpDomain>{OrtCreateCustomOpDomain(domain)} {
}
inline void CustomOpDomain::Add(OrtCustomOp* op) {
ORT_THROW_ON_ERROR(OrtCustomOpDomain_Add(p_, op));
}
inline RunOptions::RunOptions() : Base<OrtRunOptions>{
OrtCreateRunOptions()} {}
inline RunOptions& RunOptions::SetRunLogVerbosityLevel(unsigned int level) {
ORT_THROW_ON_ERROR(OrtRunOptionsSetRunLogVerbosityLevel(p_, level));
return *this;
}
inline unsigned int RunOptions::GetRunLogVerbosityLevel() const {
return OrtRunOptionsGetRunLogVerbosityLevel(p_);
}
inline RunOptions& RunOptions::SetRunTag(const char* run_tag) {
ORT_THROW_ON_ERROR(OrtRunOptionsSetRunTag(p_, run_tag));
return *this;
}
inline const char* RunOptions::GetRunTag() const {
return OrtRunOptionsGetRunTag(p_);
}
inline RunOptions& RunOptions::SetTerminate(bool flag) {
OrtRunOptionsSetTerminate(p_, flag ? 1 : 0);
return *this;
}
inline SessionOptions::SessionOptions() : Base<OrtSessionOptions>{OrtCreateSessionOptions()} {
}
inline SessionOptions SessionOptions::clone() const {
return SessionOptions{OrtCloneSessionOptions(p_)};
}
inline SessionOptions& SessionOptions::SetThreadPoolSize(int session_thread_pool_size) {
if (OrtSetSessionThreadPoolSize(p_, session_thread_pool_size) == -1)
throw Exception("Error calling SessionOptions::SetThreadPoolSize", ORT_FAIL);
return *this;
}
inline SessionOptions& SessionOptions::SetGraphOptimizationLevel(uint32_t graph_optimization_level) {
if (OrtSetSessionGraphOptimizationLevel(p_, graph_optimization_level) == -1)
throw Exception("Error calling SessionOptions::SetGraphOptimizationLevel", ORT_FAIL);
return *this;
}
inline SessionOptions& SessionOptions::EnableCpuMemArena() {
OrtEnableCpuMemArena(p_);
return *this;
}
inline SessionOptions& SessionOptions::DisableCpuMemArena() {
OrtDisableCpuMemArena(p_);
return *this;
}
inline SessionOptions& SessionOptions::EnableSequentialExecution() {
OrtEnableSequentialExecution(p_);
return *this;
}
inline SessionOptions& SessionOptions::DisableSequentialExecution() {
OrtDisableSequentialExecution(p_);
return *this;
}
inline SessionOptions& SessionOptions::SetLogId(const char* logid) {
OrtSetSessionLogId(p_, logid);
return *this;
}
inline SessionOptions& SessionOptions::Add(OrtCustomOpDomain* custom_op_domain) {
ORT_THROW_ON_ERROR(OrtAddCustomOpDomain(p_, custom_op_domain));
return *this;
}
inline Session::Session(OrtEnv* env, const ORTCHAR_T* model_path, const OrtSessionOptions* options) {
ORT_THROW_ON_ERROR(OrtCreateSession(env, model_path, options, &p_));
}
template <unsigned InputCount>
inline Value Session::Run(OrtRunOptions* run_options, const char* const* input_names, Value (&inputs)[InputCount],
const char* const* output_names, size_t output_names_len) {
std::array<OrtValue*, InputCount> internal_inputs;
std::copy_n(inputs, InputCount, internal_inputs.data());
OrtValue* out{};
ORT_THROW_ON_ERROR(OrtRun(p_, run_options, input_names, internal_inputs.data(), InputCount, output_names, output_names_len, &out));
return Value{out};
}
inline size_t Session::GetInputCount() const {
size_t out;
ORT_THROW_ON_ERROR(OrtSessionGetInputCount(p_, &out));
return out;
}
inline size_t Session::GetOutputCount() const {
size_t out;
ORT_THROW_ON_ERROR(OrtSessionGetOutputCount(p_, &out));
return out;
}
inline char* Session::GetInputName(size_t index, OrtAllocator* allocator) const {
char* out;
ORT_THROW_ON_ERROR(OrtSessionGetInputName(p_, index, allocator, &out));
return out;
}
inline char* Session::GetOutputName(size_t index, OrtAllocator* allocator) const {
char* out;
ORT_THROW_ON_ERROR(OrtSessionGetOutputName(p_, index, allocator, &out));
return out;
}
inline TypeInfo Session::GetInputTypeInfo(size_t index) const {
OrtTypeInfo* out;
ORT_THROW_ON_ERROR(OrtSessionGetInputTypeInfo(p_, index, &out));
return TypeInfo{out};
}
inline TypeInfo Session::GetOutputTypeInfo(size_t index) const {
OrtTypeInfo* out;
ORT_THROW_ON_ERROR(OrtSessionGetOutputTypeInfo(p_, index, &out));
return TypeInfo{out};
}
inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const {
return OrtGetTensorElementType(p_);
}
inline size_t TensorTypeAndShapeInfo::GetDimensionsCount() const {
return OrtGetNumOfDimensions(p_);
}
inline void TensorTypeAndShapeInfo::GetDimensions(int64_t* values, size_t values_count) const {
OrtGetDimensions(p_, values, values_count);
}
inline std::vector<int64_t> TensorTypeAndShapeInfo::GetShape() const {
std::vector<int64_t> output;
output.resize(GetDimensionsCount());
GetDimensions(output.data(), output.size());
return output;
}
inline Unowned<TensorTypeAndShapeInfo> TypeInfo::GetTensorTypeAndShapeInfo() const {
return Unowned<TensorTypeAndShapeInfo>{const_cast<OrtTensorTypeAndShapeInfo*>(OrtCastTypeInfoToTensorInfo(p_))};
}
inline Value Value::CreateTensor(const OrtAllocatorInfo* info, void* p_data, size_t p_data_len, const int64_t* shape, size_t shape_len,
ONNXTensorElementDataType type) {
OrtValue* out;
ORT_THROW_ON_ERROR(OrtCreateTensorWithDataAsOrtValue(info, p_data, p_data_len, shape, shape_len, type, &out));
return Value(out);
}
inline bool Value::IsTensor() const {
return OrtIsTensor(p_) != 0;
}
template <typename T>
T* Value::GetTensorMutableData() {
T* output;
ORT_THROW_ON_ERROR(OrtGetTensorMutableData(p_, (void**)&output));
return output;
}
inline TensorTypeAndShapeInfo Value::GetTensorTypeAndShapeInfo() const {
OrtTensorTypeAndShapeInfo* output;
ORT_THROW_ON_ERROR(OrtGetTensorShapeAndType(p_, &output));
return TensorTypeAndShapeInfo{output};
}
} // namespace Ort
// Deprecated: Will be removed once all dependencies of it are removed
#if 1
namespace onnxruntime {
class SessionOptionsWrapper {
private:
std::unique_ptr<OrtSessionOptions> value;
@ -115,20 +479,14 @@ class SessionOptionsWrapper {
OrtSessionOptions* p = OrtCloneSessionOptions(value.get());
return SessionOptionsWrapper(env_, p);
}
#ifdef _WIN32
OrtSession* OrtCreateSession(_In_ const wchar_t* model_path) {
OrtSession* OrtCreateSession(_In_ const ORTCHAR_T* model_path) {
OrtSession* ret = nullptr;
ORT_THROW_ON_ERROR(::OrtCreateSession(env_, model_path, value.get(), &ret));
return ret;
}
#else
OrtSession* OrtCreateSession(_In_ const char* model_path) {
OrtSession* ret = nullptr;
ORT_THROW_ON_ERROR(::OrtCreateSession(env_, model_path, value.get(), &ret));
return ret;
}
#endif
};
inline OrtValue* OrtCreateTensorAsOrtValue(_Inout_ OrtAllocator* env, const std::vector<int64_t>& shape, ONNXTensorElementDataType type) {
OrtValue* ret;
ORT_THROW_ON_ERROR(::OrtCreateTensorAsOrtValue(env, shape.data(), shape.size(), type, &ret));
@ -148,6 +506,10 @@ inline std::vector<int64_t> GetTensorShape(const OrtTensorTypeAndShapeInfo* info
return ret;
}
} // namespace onnxruntime
#endif
namespace Ort {
struct CustomOpApi {
CustomOpApi(const OrtCustomOpApi& api) : api_(api) {}
@ -183,11 +545,16 @@ struct CustomOpApi {
return data;
}
template <typename T>
const T* GetTensorData(_Inout_ const OrtValue* value) {
return GetTensorMutableData<T>(const_cast<OrtValue*>(value));
}
void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input) {
api_.ReleaseTensorTypeAndShapeInfo(input);
}
OrtValue* KernelContext_GetInput(OrtKernelContext* context, _In_ size_t index) {
const OrtValue* KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index) {
return api_.KernelContext_GetInput(context, index);
}
OrtValue* KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count) {
@ -231,6 +598,6 @@ struct CustomOpBase : OrtCustomOp {
}
};
} // namespace onnxruntime
} // namespace Ort
#undef ORT_REDIRECT_SIMPLE_FUNCTION_CALL

5
onnxruntime/core/session/custom_ops.cc
View file

@ -29,8 +29,8 @@ ORT_API_STATUS_IMPL(OrtKernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* i
return onnxruntime::ToOrtStatus(status);
}
ORT_API(OrtValue*, OrtKernelContext_GetInput, OrtKernelContext* context, _In_ size_t index) {
return reinterpret_cast<OrtValue*>(const_cast<onnxruntime::MLValue*>(reinterpret_cast<onnxruntime::OpKernelContextInternal*>(context)->GetInputMLValue(index)));
ORT_API(const OrtValue*, OrtKernelContext_GetInput, const OrtKernelContext* context, _In_ size_t index) {
return reinterpret_cast<const OrtValue*>(reinterpret_cast<const onnxruntime::OpKernelContextInternal*>(context)->GetInputMLValue(index));
};
ORT_API(OrtValue*, OrtKernelContext_GetOutput, OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count) {
@ -58,7 +58,6 @@ constexpr OrtCustomOpApi g_custom_op_api = {
};
namespace onnxruntime {
struct CustomOpKernel : OpKernel {
CustomOpKernel(const OpKernelInfo& info, OrtCustomOp& op) : OpKernel(info), op_(op) {
if (op_.version != 1)

1
onnxruntime/test/onnx/TestCase.h
View file

@ -5,6 +5,7 @@
#include <vector>
#include <mutex>
#include <unordered_map>
#include <core/common/common.h>
#include <core/common/status.h>
#include <core/session/onnxruntime_cxx_api.h>
#include <core/framework/path_lib.h>

34
onnxruntime/test/onnx/main.cc
View file

@ -67,9 +67,9 @@ int GetNumCpuCores() { return std::thread::hardware_concurrency(); }
} // namespace
#ifdef _WIN32
int real_main(int argc, wchar_t* argv[], OrtEnv** p_env) {
int real_main(int argc, wchar_t* argv[], Ort::Env& env) {
#else
int real_main(int argc, char* argv[], OrtEnv** p_env) {
int real_main(int argc, char* argv[], Ort::Env& env) {
#endif
// if this var is not empty, only run the tests with name in this list
std::vector<std::basic_string<PATH_CHAR_TYPE> > whitelisted_test_cases;
@ -164,16 +164,14 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
usage();
return -1;
}
OrtEnv* env;
{
OrtStatus* ost = OrtCreateEnv(logging_level, "Default", &env);
if (ost != nullptr) {
fprintf(stderr, "Error creating environment: %s \n", OrtGetErrorMessage(ost));
OrtReleaseStatus(ost);
return -1;
}
*p_env = env;
try {
env = Ort::Env{logging_level, "Default"};
} catch (std::exception& ex) {
fprintf(stderr, "Error creating environment: %s \n", ex.what());
return -1;
}
std::vector<std::basic_string<PATH_CHAR_TYPE> > data_dirs;
TestResultStat stat;
@ -184,7 +182,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
double per_sample_tolerance = 1e-3;
// when cuda is enabled, set it to a larger value for resolving random MNIST test failure
double relative_per_sample_tolerance = enable_cuda ? 0.017 : 1e-3;
SessionOptionsWrapper sf(env);
Ort::SessionOptions sf;
if (enable_cpu_mem_arena)
sf.EnableCpuMemArena();
else
@ -265,7 +263,8 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
}
}
}
TestEnv args(tests, stat, sf);
TestEnv args(tests, stat, env, sf);
Status st = RunTests(args, p_models, concurrent_session_runs, static_cast<size_t>(repeat_count),
GetDefaultThreadPool(Env::Default()));
if (!st.IsOK()) {
@ -431,17 +430,16 @@ int wmain(int argc, wchar_t* argv[]) {
#else
int main(int argc, char* argv[]) {
#endif
OrtEnv* env = nullptr;
Ort::Env env{nullptr};
int retval = -1;
try {
retval = real_main(argc, argv, &env);
retval = real_main(argc, argv, env);
} catch (std::exception& ex) {
fprintf(stderr, "%s\n", ex.what());
retval = -1;
}
if (env) {
OrtReleaseEnv(env);
} else {
// Release the protobuf library if we failed to create an env (the env will release it automatically on destruction)
if (!env) {
::google::protobuf::ShutdownProtobufLibrary();
}
return retval;

21
onnxruntime/test/onnx/runner.cc
View file

@ -34,7 +34,7 @@ void ORT_CALLBACK RunTestCase(ORT_CALLBACK_INSTANCE pci, void* context, ORT_WORK
ITestCase* info = task->env.tests[task->task_id];
std::shared_ptr<TestCaseResult> ret;
try {
RunSingleTestCase(info, task->env.sf, task->concurrent_runs, task->repeat_count, task->pool, pci, [task](std::shared_ptr<TestCaseResult> result, ORT_CALLBACK_INSTANCE pci) {
RunSingleTestCase(info, task->env.env, task->env.sf, task->concurrent_runs, task->repeat_count, task->pool, pci, [task](std::shared_ptr<TestCaseResult> result, ORT_CALLBACK_INSTANCE pci) {
return OnTestCaseFinished(pci, task, result);
});
return;
@ -169,7 +169,7 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c
ORT_EVENT ev;
ORT_RETURN_IF_ERROR(CreateOnnxRuntimeEvent(&ev));
try {
RunSingleTestCase(env.tests[i], env.sf, concurrent_runs, repeat_count, tpool, nullptr, [repeat_count, &results, ev, concurrent_runs, test_case_name](std::shared_ptr<TestCaseResult> result, ORT_CALLBACK_INSTANCE pci) {
RunSingleTestCase(env.tests[i], env.env, env.sf, concurrent_runs, repeat_count, tpool, nullptr, [repeat_count, &results, ev, concurrent_runs, test_case_name](std::shared_ptr<TestCaseResult> result, ORT_CALLBACK_INSTANCE pci) {
//TODO:output this information to a xml
if (concurrent_runs == 1) {
TIME_SPEC ts = result->GetSpentTime();
@ -310,6 +310,7 @@ std::pair<COMPARE_RESULT, std::string> CompareGenericValue(const OrtValue* o, co
bool post_processing) {
return onnxruntime::CompareMLValue(*(MLValue*)o, *(MLValue*)expected_mlvalue, per_sample_tolerance, relative_per_sample_tolerance, post_processing);
}
EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) {
HeapBuffer holder;
std::unordered_map<std::string, OrtValue*> feeds;
@ -400,7 +401,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) {
if (compare_result == COMPARE_RESULT::SUCCESS) {
const ONNX_NAMESPACE::ValueInfoProto* v = name_output_value_info_proto[output_name];
if (v == nullptr) continue;
ret = VerifyValueInfo(*v, actual_output_value);
ret = VerifyValueInfo(*v, Ort::Unowned<Ort::Value>{actual_output_value});
compare_result = ret.first;
if (compare_result != COMPARE_RESULT::SUCCESS) {
switch (compare_result) {
@ -458,16 +459,15 @@ void SeqTestRunner::Start(ORT_CALLBACK_INSTANCE pci, size_t) {
finish(pci);
}
void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ORT_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished) {
void RunSingleTestCase(ITestCase* info, Ort::Env& env, const Ort::SessionOptions& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ORT_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished) {
std::shared_ptr<TestCaseResult> ret;
size_t data_count = info->GetDataCount();
try {
DataRunner* r = nullptr;
std::string node_name = info->GetNodeName();
auto sf2 = sf.clone();
sf2.SetSessionLogId(info->GetTestCaseName().c_str());
std::unique_ptr<OrtSession, decltype(&OrtReleaseSession)> session_object(
sf2.OrtCreateSession(info->GetModelUrl()), OrtReleaseSession);
sf2.SetLogId(info->GetTestCaseName().c_str());
Ort::Session session_object{env, info->GetModelUrl(), sf2};
LOGF_DEFAULT(INFO, "testing %s\n", info->GetTestCaseName().c_str());
//temp hack. Because we have no resource control. We may not have enough memory to run this test in parallel
if (info->GetTestCaseName() == "coreml_FNS-Candy_ImageNet")
@ -479,6 +479,13 @@ void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper
}
r->Start(pci, concurrent_runs);
return;
} catch (const Ort::Exception& ex) {
if (ex.GetOrtErrorCode() != ORT_NOT_IMPLEMENTED)
throw;
LOGF_DEFAULT(ERROR, "Test %s failed:%s", info->GetTestCaseName().c_str(), ex.what());
std::string node_name;
ret = std::make_shared<TestCaseResult>(data_count, EXECUTE_RESULT::NOT_SUPPORT, "");
} catch (onnxruntime::NotImplementedException& ex) {
LOGF_DEFAULT(ERROR, "Test %s failed:%s", info->GetTestCaseName().c_str(), ex.what());
std::string node_name;

2
onnxruntime/test/onnx/runner.h
View file

@ -136,5 +136,5 @@ void LoadTests(const std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_paths
//Do not run this function in the thread pool passed in
::onnxruntime::common::Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_count, PThreadPool tpool);
EXECUTE_RESULT StatusCodeToExecuteResult(int input);
void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper& sf, size_t concurrent_runs,
void RunSingleTestCase(ITestCase* info, Ort::Env& env, const Ort::SessionOptions& sf, size_t concurrent_runs,
size_t repeat_count, PThreadPool tpool, ORT_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished);

6
onnxruntime/test/onnx/testenv.cc
View file

@ -5,11 +5,9 @@
#include "FixedCountFinishCallback.h"
#include <core/session/onnxruntime_cxx_api.h>
using onnxruntime::SessionOptionsWrapper;
using onnxruntime::Status;
TestEnv::TestEnv(const std::vector<ITestCase*>& tests1, TestResultStat& stat1, SessionOptionsWrapper& sf1)
: tests(tests1), next_test_to_run(0), stat(stat1), finished(new FixedCountFinishCallback(static_cast<int>(tests1.size()))), sf(sf1) {
TestEnv::TestEnv(const std::vector<ITestCase*>& tests1, TestResultStat& stat1, Ort::Env& env1, Ort::SessionOptions& sf1)
: tests(tests1), next_test_to_run(0), stat(stat1), finished(new FixedCountFinishCallback(static_cast<int>(tests1.size()))), env(env1), sf(sf1) {
}
TestEnv::~TestEnv() {

5
onnxruntime/test/onnx/testenv.h
View file

@ -20,8 +20,9 @@ class TestEnv {
std::atomic_int next_test_to_run;
TestResultStat& stat;
FixedCountFinishCallback* finished;
const onnxruntime::SessionOptionsWrapper& sf;
TestEnv(const std::vector<ITestCase*>& tests, TestResultStat& stat1, onnxruntime::SessionOptionsWrapper& sf1);
Ort::Env& env;
const Ort::SessionOptions& sf;
TestEnv(const std::vector<ITestCase*>& tests, TestResultStat& stat1, Ort::Env& env, Ort::SessionOptions& sf1);
~TestEnv();
private:

15
onnxruntime/test/shared_lib/test_allocator.cc
View file

@ -18,17 +18,12 @@ TEST_F(CApiTest, allocation_info) {
}
TEST_F(CApiTest, DefaultAllocator) {
std::unique_ptr<OrtAllocator> default_allocator;
{
OrtAllocator* ptr;
ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&ptr));
default_allocator.reset(ptr);
}
char* p = (char*)OrtAllocatorAlloc(default_allocator.get(), 100);
Ort::Allocator default_allocator = Ort::Allocator::Create_Default();
char* p = (char*)default_allocator.Alloc(100);
ASSERT_NE(p, nullptr);
memset(p, 0, 100);
OrtAllocatorFree(default_allocator.get(), p);
const OrtAllocatorInfo* info1 = OrtAllocatorGetInfo(default_allocator.get());
const OrtAllocatorInfo* info2 = default_allocator->Info(default_allocator.get());
default_allocator.Free(p);
const OrtAllocatorInfo* info1 = default_allocator.GetInfo();
const OrtAllocatorInfo* info2 = static_cast<OrtAllocator*>(default_allocator)->Info(default_allocator);
ASSERT_EQ(0, OrtCompareAllocatorInfo(info1, info2));
}

20
onnxruntime/test/shared_lib/test_inference.cc
View file

@ -166,7 +166,7 @@ INSTANTIATE_TEST_CASE_P(CApiTestWithProviders,
::testing::Values(0, 1, 2, 3, 4));
struct OrtTensorDimensions : std::vector<int64_t> {
OrtTensorDimensions(onnxruntime::CustomOpApi ort, OrtValue* value) {
OrtTensorDimensions(Ort::CustomOpApi ort, const OrtValue* value) {
OrtTensorTypeAndShapeInfo* info = ort.GetTensorShapeAndType(value);
auto dimensionCount = ort.GetDimensionCount(info);
resize(dimensionCount);
@ -187,21 +187,21 @@ template <typename T, size_t N>
constexpr size_t countof(T (&)[N]) { return N; }
struct MyCustomKernel {
MyCustomKernel(onnxruntime::CustomOpApi ort, const OrtKernelInfo* /*info*/) : ort_(ort) {
MyCustomKernel(Ort::CustomOpApi ort, const OrtKernelInfo* /*info*/) : ort_(ort) {
}
void GetOutputShape(OrtKernelContext* context, size_t /*output_index*/, OrtTensorTypeAndShapeInfo* info) {
OrtValue* input_X = ort_.KernelContext_GetInput(context, 0);
const OrtValue* input_X = ort_.KernelContext_GetInput(context, 0);
OrtTensorDimensions dimensions(ort_, input_X);
ort_.SetDimensions(info, dimensions.data(), dimensions.size());
}
void Compute(OrtKernelContext* context) {
// Setup inputs
OrtValue* input_X = ort_.KernelContext_GetInput(context, 0);
OrtValue* input_Y = ort_.KernelContext_GetInput(context, 1);
float* X = ort_.GetTensorMutableData<float>(input_X);
float* Y = ort_.GetTensorMutableData<float>(input_Y);
const OrtValue* input_X = ort_.KernelContext_GetInput(context, 0);
const OrtValue* input_Y = ort_.KernelContext_GetInput(context, 1);
const float* X = ort_.GetTensorData<float>(input_X);
const float* Y = ort_.GetTensorData<float>(input_Y);
// Setup output
OrtTensorDimensions dimensions(ort_, input_X);
@ -219,11 +219,11 @@ struct MyCustomKernel {
}
private:
onnxruntime::CustomOpApi ort_;
Ort::CustomOpApi ort_;
};
struct MyCustomOp : onnxruntime::CustomOpBase<MyCustomOp, MyCustomKernel> {
void* CreateKernel(onnxruntime::CustomOpApi api, const OrtKernelInfo* info) { return new MyCustomKernel(api, info); };
struct MyCustomOp : Ort::CustomOpBase<MyCustomOp, MyCustomKernel> {
void* CreateKernel(Ort::CustomOpApi api, const OrtKernelInfo* info) { return new MyCustomKernel(api, info); };
const char* GetName() const { return "Foo"; };
size_t GetInputTypeCount() const { return 2; };

11
onnxruntime/test/shared_lib/test_run_options.cc
View file

@ -3,13 +3,12 @@
#include "core/session/onnxruntime_cxx_api.h"
#include "test_fixture.h"
using namespace onnxruntime;
TEST_F(CApiTest, run_options) {
std::unique_ptr<OrtRunOptions> options(OrtCreateRunOptions());
Ort::RunOptions options;
ASSERT_NE(options, nullptr);
ASSERT_EQ(OrtRunOptionsSetRunLogVerbosityLevel(options.get(), 1), nullptr);
ASSERT_EQ(OrtRunOptionsSetRunTag(options.get(), "abc"), nullptr);
ASSERT_STREQ(OrtRunOptionsGetRunTag(options.get()), "abc");
ASSERT_EQ(OrtRunOptionsGetRunLogVerbosityLevel(options.get()), unsigned(1));
options.SetRunLogVerbosityLevel(1);
options.SetRunTag("abc");
ASSERT_STREQ(options.GetRunTag(), "abc");
ASSERT_EQ(options.GetRunLogVerbosityLevel(), unsigned(1));
}

1
onnxruntime/test/shared_lib/test_tensor_loader.cc
View file

@ -2,6 +2,7 @@
// Licensed under the MIT License.
#include "core/session/onnxruntime_cxx_api.h"
#include "core/common/common.h"
#include "onnx_protobuf.h"
#include "test_fixture.h"

17
onnxruntime/test/util/compare_mlvalue.cc
View file

@ -341,10 +341,10 @@ std::pair<COMPARE_RESULT, std::string> CompareMLValue(const MLValue& o, const ML
per_sample_tolerance, relative_per_sample_tolerance, post_processing);
}
std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const OrtValue* o) {
std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const Ort::Value& o) {
if (!v.has_type()) return std::make_pair(COMPARE_RESULT::SUCCESS, "");
if (v.type().has_tensor_type()) {
if (!OrtIsTensor(o)) {
if (!o.IsTensor()) {
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, "");
}
@ -353,13 +353,8 @@ std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::Val
//if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) {
// return COMPARE_RESULT::TYPE_MISMATCH;
//}
std::unique_ptr<OrtTensorTypeAndShapeInfo> info;
{
OrtTensorTypeAndShapeInfo* t1 = nullptr;
ORT_THROW_ON_ERROR(OrtGetTensorShapeAndType(o, &t1));
info.reset(t1);
}
ONNXTensorElementDataType real_type = OrtGetTensorElementType(info.get());
auto info = o.GetTensorTypeAndShapeInfo();
ONNXTensorElementDataType real_type = info.GetElementType();
ONNXTensorElementDataType expected_type = onnxruntime::utils::CApiElementTypeFromProtoType(t.elem_type());
if (real_type != expected_type) {
std::ostringstream oss;
@ -368,7 +363,7 @@ std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::Val
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str());
}
std::vector<int64_t> shape = GetTensorShape(info.get());
std::vector<int64_t> shape = info.GetShape();
const auto& tensor_shape_proto = t.shape();
if (!AreShapesEqual(shape, tensor_shape_proto)) {
std::ostringstream oss;
@ -386,7 +381,7 @@ std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::Val
//Cannot do this check for tensor type.
//For tensor type, o.Type() is TensorTypeBase*, but p points to a subclass of TensorTypeBase
auto p = DataTypeImpl::TypeFromProto(v.type());
if (((MLValue*)o)->Type() != p) {
if (((MLValue*)(const OrtValue*)o)->Type() != p) {
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, "");
}
}

2
onnxruntime/test/util/include/test/compare_mlvalue.h
View file

@ -23,5 +23,5 @@ std::pair<COMPARE_RESULT, std::string> CompareMLValue(const MLValue& real, const
double relative_per_sample_tolerance, bool post_processing);
//verify if the 'value' matches the 'expected' ValueInfoProto. 'value' is a model output
std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const OrtValue* value);
std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const Ort::Value& value);
} // namespace onnxruntime