saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve TensorRT GetCapability to Enable More Models (#1012)

Browse source 
* Improve TensorRT GetCapability Accuracy

* Update onnxruntime_providers.cmake

* made changes based on feedback

* update unit tests for TensorRT

* update onnx-tensorrt submodule to v5.0 branch

* remove uncessary comments

* convert int32 to int64 at inferencing output

* add more data types in compute

* change returns in compute

* use StatusCode as return in compute
This commit is contained in:
stevenlix 2019-05-24 10:12:55 -07:00 committed by GitHub
parent b44a30bca7
commit 723d5c782a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 462 additions and 291 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitmodules vendored
View file

@ -28,6 +28,7 @@
[submodule "cmake/external/onnx-tensorrt"]
path = cmake/external/onnx-tensorrt
url = https://github.com/onnx/onnx-tensorrt.git
branch = v5.0
[submodule "cmake/external/eigen"]
path = cmake/external/eigen
url = https://github.com/eigenteam/eigen-git-mirror.git

2
cmake/external/onnx-tensorrt vendored

@ -1 +1 @@
Subproject commit c8ebe0da0776c1a6347139b074a901bd57fae499
Subproject commit 3aa0a1cb41fae88b7787b6289a729ed9046a18e4

2
cmake/onnxruntime_providers.cmake
View file

@ -123,7 +123,7 @@ if (onnxruntime_USE_TENSORRT)
set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
if (WIN32)
set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd2220")
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4701 /wd4805")
endif()

367
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -10,6 +10,7 @@
#include "core/framework/compute_capability.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/platform/env.h"
#include "core/common/status.h"
#include "onnx/shape_inference/implementation.h"
#include "cuda_runtime_api.h"
#include "gsl/pointers"
@ -46,16 +47,200 @@ TensorrtExecutionProvider::TensorrtExecutionProvider()
TensorrtExecutionProvider::~TensorrtExecutionProvider() {}
std::unique_ptr<IndexedSubGraph> TensorrtExecutionProvider::GetSubGraph(SubGraph_t graph_nodes_index, int& kernels_index, const onnxruntime::GraphViewer& graph) const {
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
std::unordered_set<size_t> node_set;
node_set.reserve(graph_nodes_index.first.size());
for (const auto& index : graph_nodes_index.first) {
node_set.insert(node_index[index]);
}
std::unique_ptr<IndexedSubGraph> sub_graph = std::make_unique<IndexedSubGraph>();
// Find inputs and outputs of the subgraph
std::unordered_map<const NodeArg *, int> fused_inputs, fused_outputs, fused_outputs_to_add;
std::unordered_set<const NodeArg*> erased;
int input_order = 0;
int output_order = 0;
for (const auto& index : graph_nodes_index.first) {
sub_graph->nodes.push_back(node_index[index]);
const auto& node = graph.GetNode(node_index[index]);
for (const auto& input : node->InputDefs()) {
const auto& it = fused_outputs.find(input);
if (it != fused_outputs.end()) {
fused_outputs.erase(it);
erased.insert(input);
} else if (erased.find(input) == erased.end()) {
//only when input is neither in output list nor erased list, add the input to input list
fused_inputs[input] = input_order++;
}
}
// For output searching, there is a special case:
// If node's OutputEdges are more than its outputs, meaning certain output is used more than once,
// if the output is connected to nodes that don't belong to the subgraph, the output need to be added
// to the output list
if (node->GetOutputEdgesCount() > node->OutputDefs().size()) {
for (auto it = node->OutputEdgesBegin(), end = node->OutputEdgesEnd(); it != end; ++it) {
const auto& node_idx = it->GetNode().Index();
const auto& output = (it->GetNode()).InputDefs()[it->GetDstArgIndex()];
if (node_set.find(node_idx) != node_set.end()) {
const auto& iter = fused_inputs.find(output);
if (iter != fused_inputs.end()) {
fused_inputs.erase(iter);
erased.insert(output);
} else if (erased.find(output) == erased.end()) {
fused_outputs[output] = output_order++;
}
} else {
fused_outputs_to_add[output] = output_order++;
}
}
} else {
for (const auto& output : node->OutputDefs()) {
const auto& it = fused_inputs.find(output);
if (it != fused_inputs.end()) {
fused_inputs.erase(it);
erased.insert(output);
}
// only when output is neither in input list nor erased list, add the output to output list
else if (erased.find(output) == erased.end()) {
fused_outputs[output] = output_order++;
}
}
}
}
fused_outputs.insert(fused_outputs_to_add.begin(), fused_outputs_to_add.end());
// Sort inputs and outputs by the order they were added
std::multimap<int, const NodeArg *> inputs, outputs;
for (auto it = fused_inputs.begin(), end = fused_inputs.end(); it != end; ++it) {
inputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
}
for (auto it = fused_outputs.begin(), end = fused_outputs.end(); it != end; ++it) {
outputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
}
// Assign inputs and outputs to subgraph's meta_def
auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>();
meta_def->name = "TRTKernel_" + std::to_string(kernels_index++);
meta_def->domain = kMSDomain;
for (const auto& input : inputs) {
meta_def->inputs.push_back(input.second->Name());
}
for (const auto& output : outputs) {
meta_def->outputs.push_back(output.second->Name());
}
meta_def->since_version = 1;
sub_graph->SetMetaDef(meta_def);
return sub_graph;
}
SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollection_t nodes_vector_input, int iterations, const int max_iterations,
const onnxruntime::GraphViewer& graph, bool* early_termination) const {
// Return if iterations are exceeding predefined number
SubGraphCollection_t nodes_list_output;
if (iterations > max_iterations) {
*early_termination = true;
return nodes_list_output;
}
iterations++;
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
int counter = 0;
for (const auto& group : nodes_vector_input) {
//construct subgraph
if (!group.first.empty()) {
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(group, counter, graph);
if (group.second) {
nodes_list_output.push_back(group);
} else {
onnxruntime::Model model_build(graph.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap());
onnxruntime::Graph& graph_build = model_build.MainGraph();
//Add node and node args
for (const auto& index : group.first) {
const auto& node = graph.GetNode(node_index[index]);
std::vector<onnxruntime::NodeArg *> inputs, outputs;
for (auto input : node->InputDefs()) {
auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto());
inputs.push_back(&n_input);
}
for (auto output : node->OutputDefs()) {
auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto());
outputs.push_back(&n_output);
}
graph_build.AddNode(node->Name(), node->OpType(), node->Description(), inputs, outputs, &node->GetAttributes(), node->Domain());
}
for (const auto& input : sub_graph->GetMetaDef()->inputs) {
const ONNX_NAMESPACE::TensorProto* initializer = nullptr;
if (graph.GetInitializedTensor(input, initializer)) {
graph_build.AddInitializedTensor(*initializer);
}
}
ORT_ENFORCE(graph_build.Resolve().IsOK());
// Serialize modelproto to string
ONNX_NAMESPACE::ModelProto model_proto = model_build.ToProto();
string string_buf;
model_proto.SerializeToString(&string_buf);
// Get supported node list recursively
SubGraphCollection_t parser_nodes_list;
TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_list);
SubGraphCollection_t next_nodes_list;
const onnxruntime::GraphViewer graph_viewer(graph_build);
next_nodes_list = GetSupportedList(parser_nodes_list, iterations, max_iterations, graph_viewer, early_termination);
for (int i = 0, end = next_nodes_list.size(); i < end; ++i) {
for (int j = 0, end = next_nodes_list[i].first.size(); j < end; ++j) {
next_nodes_list[i].first[j] = group.first[next_nodes_list[i].first[j]];
}
nodes_list_output.push_back(next_nodes_list[i]);
}
}
}
}
return nodes_list_output;
}
std::vector<std::unique_ptr<ComputeCapability>>
TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
const std::vector<const KernelRegistry*>& /*kernel_registries*/) const {
// Construct modelproto from graph
onnxruntime::Model model(graph.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap());
onnxruntime::Graph& graph_build = model.MainGraph();
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
for (const auto& node : graph.Nodes()) {
graph_build.AddNode(node);
std::vector<onnxruntime::NodeArg *> inputs, outputs;
for (auto input : node.InputDefs()) {
auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto());
inputs.push_back(&n_input);
}
for (auto output : node.OutputDefs()) {
auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto());
outputs.push_back(&n_output);
}
graph_build.AddNode(node.Name(), node.OpType(), node.Description(), inputs, outputs, &node.GetAttributes(), node.Domain());
}
//Add initializer to graph
const auto& init_tensors = graph.GetAllInitializedTensors();
for (const auto& tensor : init_tensors) {
graph_build.AddInitializedTensor(*(tensor.second));
}
ORT_ENFORCE(graph_build.Resolve().IsOK());
ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
@ -65,114 +250,28 @@ TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
model_proto.SerializeToString(&string_buf);
// Get supported node list
SubGraphCollection_t supported_nodes_vector;
SubGraphCollection_t parser_nodes_vector;
TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
trt_parser->supportsModel(string_buf.data(), string_buf.size(), supported_nodes_vector);
trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_vector);
// Find inputs, initializers and outputs for each supported subgraph
SubGraphCollection_t supported_nodes_vector;
const char* batch_env = getenv("ORT_TENSORRT_MAX_PARSER_ITERATIONS");
const int max_iterations = batch_env ? atoi(batch_env) : max_parser_iterations_;
bool early_termination = false;
supported_nodes_vector = GetSupportedList(parser_nodes_vector, 0, max_iterations, graph, &early_termination);
if (early_termination) {
supported_nodes_vector.clear();
}
// Construct subgraph capability from node list
std::vector<std::unique_ptr<ComputeCapability>> result;
int counter = 0;
for (const auto& group : supported_nodes_vector) {
if (!group.empty()) {
std::unordered_set<size_t> node_set;
node_set.reserve(group.size());
for (const auto& index : group) {
node_set.insert(node_index[index]);
}
std::unique_ptr<IndexedSubGraph> sub_graph = std::make_unique<IndexedSubGraph>();
// Find inputs and outputs of the subgraph
std::unordered_map<const NodeArg*, int> fused_inputs, fused_outputs, fused_outputs_to_add;
std::unordered_set<const NodeArg*> erased;
int input_order = 0;
int output_order = 0;
for (const auto& index : group) {
sub_graph->nodes.push_back(node_index[index]);
const auto& node = graph.GetNode(node_index[index]);
for (const auto& input : node->InputDefs()) {
const auto& it = fused_outputs.find(input);
if (it != fused_outputs.end()) {
fused_outputs.erase(it);
erased.insert(input);
}
//only when input is neither in output list nor erased list, add the input to input list
else if (erased.find(input) == erased.end()) {
fused_inputs[input] = input_order++;
}
}
// For output searching, there is a special case:
// If node's OutputEdges are more than its outputs, meaning certain output is used more than once,
// if the output is connected to nodes that don't belong to the subgraph, the output need to be added
// to the output list
if (node->GetOutputEdgesCount() > node->OutputDefs().size()) {
for (auto it = node->OutputEdgesBegin(), end = node->OutputEdgesEnd(); it != end; ++it) {
const auto& node_idx = it->GetNode().Index();
const auto& output = (it->GetNode()).InputDefs()[it->GetDstArgIndex()];
if (node_set.find(node_idx) != node_set.end()) {
const auto& iter = fused_inputs.find(output);
if (iter != fused_inputs.end()) {
fused_inputs.erase(iter);
erased.insert(output);
} else if (erased.find(output) == erased.end()) {
fused_outputs[output] = output_order++;
}
} else {
fused_outputs_to_add[output] = output_order++;
}
}
} else {
for (const auto& output : node->OutputDefs()) {
const auto& it = fused_inputs.find(output);
if (it != fused_inputs.end()) {
fused_inputs.erase(it);
erased.insert(output);
}
// only when output is neither in input list nor erased list, add the output to output list
else if (erased.find(output) == erased.end()) {
fused_outputs[output] = output_order++;
}
}
}
}
fused_outputs.insert(fused_outputs_to_add.begin(), fused_outputs_to_add.end());
// Sort inputs and outputs by the order they were added
std::multimap<int, const NodeArg*> inputs, outputs;
for (auto it = fused_inputs.begin(), end = fused_inputs.end(); it != end; ++it) {
inputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
}
for (auto it = fused_outputs.begin(), end = fused_outputs.end(); it != end; ++it) {
outputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
}
// Assign inputs and outputs to subgraph's meta_def
auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>();
meta_def->name = "TRTKernel_" + std::to_string(counter++);
meta_def->domain = kMSDomain;
for (const auto& input : inputs) {
meta_def->inputs.push_back(input.second->Name());
}
for (const auto& output : outputs) {
meta_def->outputs.push_back(output.second->Name());
}
meta_def->since_version = 1;
sub_graph->SetMetaDef(meta_def);
if (!group.first.empty()) {
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(group, counter, graph);
result.push_back(std::make_unique<ComputeCapability>(std::move(sub_graph)));
}
}
@ -199,6 +298,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
std::vector<int> output_indexes;
std::vector<int> output_dim_sizes;
std::vector<std::vector<int64_t>> output_shapes;
std::vector<int> output_types;
// Build map from input name to its index in input definitions
std::unordered_map<std::string, int> input_map;
@ -226,11 +326,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
*(model_proto.mutable_graph()) = graph_body.ToGraphProto();
model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
// Create TensorRT engine
string string_buf;
model_proto.SerializeToString(&string_buf);
// Create TensorRT engine
TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
@ -282,6 +381,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
output_indexes.resize(num_outputs);
output_dim_sizes.resize(num_outputs);
output_shapes.resize(num_outputs);
output_types.resize(num_outputs);
for (int i = 0; i < num_outputs; ++i) {
const std::string& name = trt_network->getOutput(i)->getName();
size_t bindingIndex = trt_engine->getBindingIndex(name.c_str());
@ -298,8 +398,11 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
}
output_dim_sizes[bindingIndex] = dim_size;
const auto& graph_output = model_proto.graph().output(); //slx
const auto& tensor_shape = graph_output[i].type().tensor_type().shape();
const auto& graph_output = model_proto.graph().output();
const auto& tensor_type = graph_output[i].type().tensor_type();
output_types[bindingIndex] = tensor_type.elem_type();
const auto& tensor_shape = tensor_type.shape();
if (tensor_shape.dim_size() == 1 && output_shapes[bindingIndex].back() == 1) {
output_shapes[bindingIndex].pop_back();
}
@ -315,6 +418,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
input_info_[fused_node->Name()].push_back(input_dim_sizes);
output_info_[fused_node->Name()].push_back(output_indexes);
output_info_[fused_node->Name()].push_back(output_dim_sizes);
output_info_[fused_node->Name()].push_back(output_types);
output_shapes_[fused_node->Name()] = output_shapes;
// Create function state
@ -342,6 +446,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
const std::vector<int>& input_dim_sizes = (trt_state->input_info)[1];
const std::vector<int>& output_indexes = (trt_state->output_info)[0];
const std::vector<int>& output_dim_sizes = (trt_state->output_info)[1];
const std::vector<int>& output_types = (trt_state->output_info)[2];
std::vector<std::vector<int64_t>> output_shapes = trt_state->output_shapes;
int num_binding_inputs = input_indexes.size();
@ -357,8 +462,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_indexes[i]);
auto tensor_info = ort.GetTensorTypeAndShape(input_tensor);
const auto& tensor_shape = ort.GetTensorShape(tensor_info);
auto tensor_type = ort.GetTensorElementType(tensor_info);
ort.ReleaseTensorTypeAndShapeInfo(tensor_info);
const float* input = ort.GetTensorData<float>(input_tensor);
const int input_batch_size = tensor_shape[0];
if (i > 0 && batch_size != input_batch_size) {
@ -366,13 +471,35 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
}
batch_size = input_batch_size;
CHECK_CUDA(cudaMalloc(&buffers[i], input_batch_size * input_dim_sizes[i] * sizeof(float)));
CHECK_CUDA(cudaMemcpy(buffers[i], input, input_batch_size * input_dim_sizes[i] * sizeof(float), cudaMemcpyHostToDevice));
int input_size = batch_size * input_dim_sizes[i];
if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
const float* input = const_cast<float*>(ort.GetTensorData<float>(input_tensor));
CHECK_CUDA(cudaMalloc(&buffers[i], input_size * sizeof(float)));
CHECK_CUDA(cudaMemcpy(buffers[i], input, input_size * sizeof(float), cudaMemcpyHostToDevice));
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
const int8_t* input = const_cast<int8_t*>(ort.GetTensorData<int8_t>(input_tensor));
CHECK_CUDA(cudaMalloc(&buffers[i], input_size * sizeof(int8_t)));
CHECK_CUDA(cudaMemcpy(buffers[i], input, input_size * sizeof(int8_t), cudaMemcpyHostToDevice));
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
const int32_t* input = const_cast<int32_t*>(ort.GetTensorData<int32_t>(input_tensor));
CHECK_CUDA(cudaMalloc(&buffers[i], input_size * sizeof(int32_t)));
CHECK_CUDA(cudaMemcpy(buffers[i], input, input_size * sizeof(int32_t), cudaMemcpyHostToDevice));
} else {
return static_cast<int>(common::StatusCode::NOT_IMPLEMENTED);
}
}
// Allocate cuda memory for outputs
for (int i = 0, end = num_binding_outputs; i < end; ++i) {
CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float)));
if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float)));
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(int8_t)));
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 || output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) {
CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(int32_t)));
} else {
return static_cast<int>(common::StatusCode::NOT_IMPLEMENTED);
}
}
// Run TRT inference
@ -383,8 +510,26 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
for (int i = 0, end = num_binding_outputs; i < end; ++i) {
int output_index = output_indexes[i];
output_shapes[i].insert(output_shapes[i].begin(), batch_size);
int output_size = batch_size * output_dim_sizes[i];
OrtValue* output_tensor = ort.KernelContext_GetOutput(context, output_index, output_shapes[i].data(), output_shapes[i].size());
CHECK_CUDA(cudaMemcpy(ort.GetTensorMutableData<float>(output_tensor), buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float), cudaMemcpyDeviceToHost));
if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
CHECK_CUDA(cudaMemcpy(ort.GetTensorMutableData<float>(output_tensor), buffers[i + num_binding_inputs], output_size * sizeof(float), cudaMemcpyDeviceToHost));
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
CHECK_CUDA(cudaMemcpy(ort.GetTensorMutableData<int8_t>(output_tensor), buffers[i + num_binding_inputs], output_size * sizeof(int8_t), cudaMemcpyDeviceToHost));
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
CHECK_CUDA(cudaMemcpy(ort.GetTensorMutableData<int32_t>(output_tensor), buffers[i + num_binding_inputs], output_size * sizeof(int32_t), cudaMemcpyDeviceToHost));
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) {
// If output tensor type is INT64, TensorRT processes data as INT32 and the output will be converted to INT64.
int* output = new int32_t[output_size];
CHECK_CUDA(cudaMemcpy(output, buffers[i + num_binding_inputs], output_size * sizeof(int32_t), cudaMemcpyDeviceToHost));
for (int j = 0; j < output_size; ++j) {
ort.GetTensorMutableData<int64_t>(output_tensor)[j] = output[j];
}
delete[] output;
} else {
return static_cast<int>(common::StatusCode::NOT_IMPLEMENTED);
}
}
// Sync stream

59
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -12,25 +12,22 @@
namespace onnxruntime {
class TensorrtLogger : public nvinfer1::ILogger {
nvinfer1::ILogger::Severity verbosity_;
public:
TensorrtLogger(Severity verbosity=Severity::kWARNING)
: verbosity_(verbosity) {}
void log(Severity severity, const char* msg) override {
if( severity <= verbosity_ ) {
time_t rawtime = std::time(0);
char buf[256];
strftime(&buf[0], 256,
"%Y-%m-%d %H:%M:%S",
std::gmtime(&rawtime));
const char* sevstr = (severity == Severity::kINTERNAL_ERROR ? " BUG" :
severity == Severity::kERROR ? " ERROR" :
severity == Severity::kWARNING ? "WARNING" :
severity == Severity::kINFO ? " INFO" :
"UNKNOWN");
LOGS_DEFAULT(WARNING) << "[" << buf << " " << sevstr << "] " << msg;
}
nvinfer1::ILogger::Severity verbosity_;
public:
TensorrtLogger(Severity verbosity = Severity::kWARNING)
: verbosity_(verbosity) {}
void log(Severity severity, const char* msg) override {
if (severity <= verbosity_) {
time_t rawtime = std::time(0);
char buf[256];
strftime(&buf[0], 256,
"%Y-%m-%d %H:%M:%S",
std::gmtime(&rawtime));
const char* sevstr = (severity == Severity::kINTERNAL_ERROR ? " BUG" : severity == Severity::kERROR ? " ERROR" : severity == Severity::kWARNING ? "WARNING" : severity == Severity::kINFO ? " INFO" : "UNKNOWN");
LOGS_DEFAULT(WARNING) << "[" << buf << " " << sevstr << "] " << msg;
}
}
};
// Information needed to construct trt execution providers.
@ -74,16 +71,17 @@ class TensorrtExecutionProvider : public IExecutionProvider {
std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
void SetMaxBatchSize(const int batch_size) {
max_batch_size_ = batch_size;
max_batch_size_ = batch_size;
}
void SetMaxWorkspaceSize(const size_t workspace_size) {
max_workspace_size_ = workspace_size;
max_workspace_size_ = workspace_size;
}
private:
int max_batch_size_ = 1;
size_t max_workspace_size_ = 1 << 30; // 1GB
int max_batch_size_ = 1;
size_t max_workspace_size_ = 1 << 30; // 1GB
int max_parser_iterations_ = 6;
struct InferDeleter {
template <typename T>
@ -105,7 +103,20 @@ class TensorrtExecutionProvider : public IExecutionProvider {
std::unordered_map<std::string, std::vector<std::vector<int>>> input_info_;
std::unordered_map<std::string, std::vector<std::vector<int>>> output_info_;
std::unordered_map<std::string, std::vector<std::vector<int64_t>>> output_shapes_;
/**Get IndexedSubGraph based on node list of the subgraph*/
std::unique_ptr<IndexedSubGraph> GetSubGraph(SubGraph_t graph_nodes_index, int& kernels_index,
const onnxruntime::GraphViewer& graph) const;
/**
Get TensorRT supported node lists by calling Onnx-TensorRT parser recursively. Since each time the parser
can only detect first unsupported node failure, it needs to wait for Onnxruntime to partition the graph
and then detect next failure again. If there are too many iterations, which means many nodes in the graph
are not supported by TensorRT, the process will be terminated and the whole graph is simply assigned to
other execution provider.
*/
SubGraphCollection_t GetSupportedList(SubGraphCollection_t supported_nodes_list, int iterations, const int max_iterations,
const onnxruntime::GraphViewer& graph, bool* early_termination) const;
};
} // namespace onnxruntime

40
onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc
View file

@ -23,7 +23,7 @@ TEST(MathOpTest, Add_int64) {
test.AddInput<int64_t>("A", {3}, {1, 2, 3});
test.AddInput<int64_t>("B", {3}, {4, 5, 6});
test.AddOutput<int64_t>("C", {3}, {5, 7, 9});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: INT64 is not supported
test.Run();
}
TEST(MathOpTest, Add) {
@ -69,7 +69,7 @@ TEST(MathOpTest, Add_Broadcast_0x0) {
test.AddInput<float>("A", {}, {10.0f});
test.AddInput<float>("B", {}, {2.0f});
test.AddOutput<float>("C", {}, {12.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: dynamic shape is not supported
test.Run();
}
TEST(MathOpTest, Add_Broadcast_0x1) {
@ -78,7 +78,7 @@ TEST(MathOpTest, Add_Broadcast_0x1) {
test.AddInput<float>("A", {}, {10.0f});
test.AddInput<float>("B", {1}, {2.0f});
test.AddOutput<float>("C", {1}, {12.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: dynamic shape is not supported
test.Run();
}
TEST(MathOpTest, Add_Broadcast_1x0) {
@ -87,7 +87,7 @@ TEST(MathOpTest, Add_Broadcast_1x0) {
test.AddInput<float>("A", {1}, {10.0f});
test.AddInput<float>("B", {}, {2.0f});
test.AddOutput<float>("C", {1}, {12.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: dynamic shape is not supported
test.Run();
}
TEST(MathOpTest, Add_Broadcast_1x1) {
@ -134,7 +134,7 @@ TEST(MathOpTest, Add_Broadcast_2x1x4_1x3x1) {
211.0f, 212.0f, 213.0f, 214.0f,
221.0f, 222.0f, 223.0f, 224.0f,
231.0f, 232.0f, 233.0f, 234.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //Input batch size is inconsistent
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Input batch size is inconsistent
}
TEST(MathOpTest, Add_Broadcast_2x1x1_3x4) {
@ -154,7 +154,7 @@ TEST(MathOpTest, Add_Broadcast_2x1x1_3x4) {
211.0f, 212.0f, 213.0f, 214.0f,
221.0f, 222.0f, 223.0f, 224.0f,
231.0f, 232.0f, 233.0f, 234.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //Input batch size is inconsistent
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Input batch size is inconsistent
}
TEST(MathOpTest, Sub_int32) {
@ -170,7 +170,7 @@ TEST(MathOpTest, Sub_int64) {
test.AddInput<int64_t>("A", {3}, {1, 5, 6});
test.AddInput<int64_t>("B", {3}, {4, 5, 3});
test.AddOutput<int64_t>("C", {3}, {-3, 0, 3});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: INT64 is not supported
test.Run();
}
TEST(MathOpTest, Sub) {
@ -203,7 +203,7 @@ TEST(MathOpTest, Sub_Broadcast_Scalar) {
{-4.0f, -3.0f, -6.0f,
-5.0f, -3.5f, -105.0f,
-10.4f, 4.3f, -10'005.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: dynamic shape is not supported
test.Run();
}
TEST(MathOpTest, Mul_int32) {
@ -219,7 +219,7 @@ TEST(MathOpTest, Mul_int64) {
test.AddInput<int64_t>("A", {3}, {3, 6, -3});
test.AddInput<int64_t>("B", {3}, {4, -3, -2});
test.AddOutput<int64_t>("C", {3}, {12, -18, 6});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: INT64 is not supported
test.Run();
}
TEST(MathOpTest, Mul) {
@ -253,7 +253,7 @@ TEST(MathOpTest, Div_int64) {
test.AddInput<int64_t>("A", {3}, {4, 8, 8});
test.AddInput<int64_t>("B", {3}, {2, 3, 4});
test.AddOutput<int64_t>("C", {3}, {2, 2, 2});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: INT64 is not supported
test.Run();
}
TEST(MathOpTest, Div) {
@ -284,7 +284,7 @@ TEST(MathOpTest, Abs_int8) {
std::vector<int64_t> dims{4};
test.AddInput<int8_t>("X", dims, {1, 2, -1, -5});
test.AddOutput<int8_t>("Y", dims, {1, 2, 1, 5});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Abs_int32) {
@ -312,7 +312,7 @@ TEST(MathOpTest, Neg_int8) {
std::vector<int64_t> dims{4};
test.AddInput<int8_t>("X", dims, {1, -2, 0, -10});
test.AddOutput<int8_t>("Y", dims, {-1, 2, 0, 10});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Neg_int32) {
@ -393,7 +393,7 @@ TEST(MathOpTest, Pow_Broadcast_Scalar0) {
test.AddInput<float>("X", {}, {2.0f});
test.AddInput<float>("Y", dims, {1.0f, 2.0f, 3.0f});
test.AddOutput<float>("Z", dims, {2.0f, 4.0f, 8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: dynamic shape is not supported
test.Run();
}
TEST(MathOpTest, Pow_Broadcast_Scalar1) {
@ -403,7 +403,7 @@ TEST(MathOpTest, Pow_Broadcast_Scalar1) {
test.AddInput<float>("X", dims, {1.0f, 2.0f, 3.0f});
test.AddInput<float>("Y", {}, {2.0f});
test.AddOutput<float>("Z", dims, {1.0f, 4.0f, 9.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: dynamic shape is not supported
test.Run();
}
TEST(MathOpTest, Exp) {
@ -416,7 +416,7 @@ TEST(MathOpTest, Exp) {
{1.0f, std::exp(1.0f),
std::exp(2.0f), std::exp(10.0f)});
test.SetOutputRelErr("Y", 1e-7f);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: result differs
}
TEST(MathOpTest, Log) {
@ -470,7 +470,7 @@ TEST(MathOpTest, Sum_8_Test1) {
311.0f, 312.0f, 313.0f,
321.0f, 322.0f, 323.0f,
331.0f, 332.0f, 333.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT parser failed on this test
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Expected output shape [{3,3,3}] did not match run output shape [{3,1,1}] for sum
}
TEST(MathOpTest, Sum_8_Test2) {
@ -499,7 +499,7 @@ TEST(MathOpTest, Sum_8_Test2) {
3.3f, 4.4f, -94.7f,
59.6f, 64.01f, -8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "Sum is not correct", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "Sum is not correct", {kTensorrtExecutionProvider}); //TensorRT: result differs
}
TEST(MathOpTest, Min_6) {
@ -582,7 +582,7 @@ TEST(MathOpTest, Max_8) {
{10.0f, 20.0f, 30.0f,
40.0f, 50.0f, 60.0f,
300.0f, 300.0f, 300.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //Input batch size is inconsistent
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Input batch size is inconsistent
}
TEST(MathOpTest, Max_8_2inputbroadcast) {
@ -597,7 +597,7 @@ TEST(MathOpTest, Max_8_2inputbroadcast) {
{10.0f, 20.0f, 30.0f,
40.0f, 50.0f, 60.0f,
70.0f, 80.0f, 90.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //Input batch size is inconsistent
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Input batch size is inconsistent
}
TEST(MathOpTest, Not) {
@ -773,7 +773,7 @@ TEST(MathOpTest, Mean_8) {
{12.0f / 3.0f, 22.0f / 3.0f, 32.0f / 3.0f,
43.0f / 3.0f, 53.0f / 3.0f, 63.0f / 3.0f,
74.0f / 3.0f, 84.0f / 3.0f, 94.0f / 3.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //Input batch size is inconsistent
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Input batch size is inconsistent
}
#ifndef DISABLE_CONTRIB_OPS

22
onnxruntime/test/providers/cpu/math/gemm_test.cc
View file

@ -7,8 +7,6 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on some of the tests because TensorRT only support FLOAT, INT8, FLOAT16 and INT32 for now
TEST(GemmOpTest, GemmNoTrans) {
OpTester test("Gemm");
@ -25,7 +23,7 @@ TEST(GemmOpTest, GemmNoTrans) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-9.0f, -9.0f, -9.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
// Only CUDA kernel has float 16 support
@ -58,7 +56,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) {
test.AddInput<MLFloat16>("B", {4, 3}, f_B);
test.AddInput<MLFloat16>("C", {2, 3}, f_C);
test.AddOutput<MLFloat16>("Y", {2, 3}, f_Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
#endif
@ -78,7 +76,7 @@ TEST(GemmOpTest, GemmBroadcast) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 12.0f, 13.0f,
-9.0f, -8.0f, -7.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GemmOpTest, GemmTrans) {
@ -99,7 +97,7 @@ TEST(GemmOpTest, GemmTrans) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-9.0f, -9.0f, -9.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GemmOpTest, GemmAlphaBeta) {
@ -118,7 +116,7 @@ TEST(GemmOpTest, GemmAlphaBeta) {
test.AddOutput<float>("Y", {2, 3},
{7.0f, 7.0f, 7.0f,
-3.0f, -3.0f, -3.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GemmOpTest, GemmNaN) {
@ -137,7 +135,7 @@ TEST(GemmOpTest, GemmNaN) {
test.AddOutput<float>("Y", {2, 3},
{10.0f, 10.0f, 10.0f,
-10.0f, -10.0f, -10.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GemmOpTest, GemmScalarBroadcast) {
@ -156,7 +154,7 @@ TEST(GemmOpTest, GemmScalarBroadcast) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-9.0f, -9.0f, -9.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Gemm2DBroadcast) {
@ -175,7 +173,7 @@ TEST(MathOpTest, Gemm2DBroadcast) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-8.0f, -8.0f, -8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GemmOpTest, GemmFalseBroadcast) {
@ -194,7 +192,7 @@ TEST(GemmOpTest, GemmFalseBroadcast) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-8.0f, -8.0f, -8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GemmOpTest, GemmEmptyTensor) {
@ -211,7 +209,7 @@ TEST(GemmOpTest, GemmEmptyTensor) {
test.AddInput<float>("C", {3}, std::vector<float>(3, 1.0f));
test.AddOutput<float>("Y", {0, 3},
{});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
} // namespace test

8
onnxruntime/test/providers/cpu/nn/pool_op_test.cc
View file

@ -51,7 +51,7 @@ TEST(PoolTest, MaxPool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: result differs
}
// Only CUDA kernel has float 16 support
@ -104,7 +104,7 @@ TEST(PoolTest, MaxPool_F16) {
test.AddInput<MLFloat16>("X", x_dims, f_X);
test.AddOutput<MLFloat16>("Y", expected_dims, f_Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Assertion `!attrs.count("pads")' failed
}
#endif
@ -569,9 +569,9 @@ TEST(PoolTest, AveragePool_10_ceil1_2d) {
test.AddAttribute("ceil_mode", (int64_t) 1);
std::vector<float> x_vals = {
1, 3, 2, 4,
1, 3, 2, 4,
5, 7, 6, 8,
9, 11, 10, 12,
9, 11, 10, 12,
13, 15, 14, 16,
};
std::vector<int64_t> x_dims = {1, 1, 4, 4};

94
onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc
View file

@ -31,7 +31,7 @@ void TestReduceOp(const std::string& op,
test.AddAttribute("keepdims", keepdims);
test.AddInput<float>("data", input_dims, data);
test.AddOutput<OutT>("reduced", expected_dims, expected_data);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider}); //TensorRT: result differs
}
TEST(ReductionOpTest, ReductionVariationTest) {
@ -69,7 +69,7 @@ TEST(ReductionOpTest, ReduceL1_default_axes_keepdims) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {78.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL1_do_not_keepdims) {
@ -96,7 +96,7 @@ TEST(ReductionOpTest, ReduceL1_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {6.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL1_keepdims) {
@ -129,7 +129,7 @@ TEST(ReductionOpTest, ReduceL1) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {33.0f, 45.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL1_int32) {
@ -145,7 +145,7 @@ TEST(ReductionOpTest, ReduceL1_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {33, 45});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL2_default_axes_keepdims) {
@ -161,7 +161,7 @@ TEST(ReductionOpTest, ReduceL2_default_axes_keepdims) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {25.49509757f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL2_do_not_keepdims) {
@ -188,7 +188,7 @@ TEST(ReductionOpTest, ReduceL2_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {3.741657387f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL2_keepdims) {
@ -222,7 +222,7 @@ TEST(ReductionOpTest, ReduceL2) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {2}, {15.71623325f, 20.07485962f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceL2_int32) {
@ -239,7 +239,7 @@ TEST(ReductionOpTest, ReduceL2_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {2}, {15, 20});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceLogSum) {
@ -280,7 +280,7 @@ TEST(ReductionOpTest, ReduceLogSum_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {1.79175947f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceLogSumExp_default_axes_keepdims) {
@ -296,7 +296,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_default_axes_keepdims) {
55.0f, 1.0f,
60.0f, 2.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {60.00671387f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims) {
@ -323,7 +323,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {3.40760596f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceLogSumExp_keepdims) {
@ -357,7 +357,7 @@ TEST(ReductionOpTest, ReduceLogSumExp) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {10.33174133f, 12.33174133f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceLogSumExp_int32) {
@ -374,7 +374,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {10, 12});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMax_default_axes_keepdims) {
@ -390,7 +390,7 @@ TEST(ReductionOpTest, ReduceMax_default_axes_keepdims) {
55.0f, 1.0f,
60.0f, 2.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {60.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMax_do_not_keepdims) {
@ -407,7 +407,7 @@ TEST(ReductionOpTest, ReduceMax_do_not_keepdims) {
55.0f, 1.0f,
60.0f, 2.0f});
test.AddOutput<float>("reduced", {3, 2}, {20.0f, 2.0f, 40.0f, 2.0f, 60.0f, 2.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMax_do_not_keepdims_2) {
@ -417,7 +417,7 @@ TEST(ReductionOpTest, ReduceMax_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{5.0f, 1.0f, 20.0f});
test.AddOutput<float>("reduced", {}, {20.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMax_keepdims) {
@ -468,7 +468,7 @@ TEST(ReductionOpTest, ReduceMax_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {3, 1, 1}, {4, 8, 12});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0
}
TEST(ReductionOpTest, ReduceMean_default_axes_keepdims) {
@ -484,7 +484,7 @@ TEST(ReductionOpTest, ReduceMean_default_axes_keepdims) {
55.0f, 1.0f,
60.0f, 2.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {18.25f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMean_do_not_keepdims) {
@ -511,7 +511,7 @@ TEST(ReductionOpTest, ReduceMean_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {2.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMean_keepdims) {
@ -545,7 +545,7 @@ TEST(ReductionOpTest, ReduceMean) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {5.5f, 7.5f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMean_int32) {
@ -562,7 +562,7 @@ TEST(ReductionOpTest, ReduceMean_int32) {
90, 100,
110, 120});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {55, 75});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMin_default_axes_keepdims) {
@ -578,7 +578,7 @@ TEST(ReductionOpTest, ReduceMin_default_axes_keepdims) {
55.0f, 1.0f,
60.0f, 2.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {1.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMin_do_not_keepdims) {
@ -605,7 +605,7 @@ TEST(ReductionOpTest, ReduceMin_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{5.0f, 1.0f, 20.0f});
test.AddOutput<float>("reduced", {}, {1.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMin_keepdims) {
@ -639,7 +639,7 @@ TEST(ReductionOpTest, ReduceMin) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {1.0f, 3.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceMin_int32) {
@ -656,7 +656,7 @@ TEST(ReductionOpTest, ReduceMin_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {1, 3});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSum) {
@ -673,7 +673,7 @@ TEST(ReductionOpTest, ReduceSum) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {33.0f, 45.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSum_axes01) {
@ -724,7 +724,7 @@ TEST(ReductionOpTest, ReduceSum_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {33, 45});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSum_default_axes_keepdims) {
@ -740,7 +740,7 @@ TEST(ReductionOpTest, ReduceSum_default_axes_keepdims) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {78.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSum_do_not_keepdims) {
@ -761,7 +761,7 @@ TEST(ReductionOpTest, ReduceSum_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {6.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSum_keepdims) {
@ -795,7 +795,7 @@ TEST(ReductionOpTest, ReduceSumSquare) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {247.0f, 403.f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSumSquare_int32) {
@ -812,7 +812,7 @@ TEST(ReductionOpTest, ReduceSumSquare_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {247, 403});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSumSquare_default_axes_keepdims) {
@ -828,7 +828,7 @@ TEST(ReductionOpTest, ReduceSumSquare_default_axes_keepdims) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {650.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSumSquare_do_not_keepdims) {
@ -855,7 +855,7 @@ TEST(ReductionOpTest, ReduceSumSquare_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {14.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceSumSquare_keepdims) {
@ -887,7 +887,7 @@ TEST(ReductionOpTest, ReduceProd_default_axes_keepdims) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 1, 1}, {479001600.f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceProd_do_not_keepdims) {
@ -914,7 +914,7 @@ TEST(ReductionOpTest, ReduceProd_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<float>("reduced", {}, {6.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceProd_keepdims) {
@ -947,7 +947,7 @@ TEST(ReductionOpTest, ReduceProd) {
9.0f, 10.0f,
11.0f, 12.0f});
test.AddOutput<float>("reduced", {1, 2, 1}, {5400.f, 88704.f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ReduceProd_int32) {
@ -963,7 +963,7 @@ TEST(ReductionOpTest, ReduceProd_int32) {
9, 10,
11, 12});
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {5400, 88704});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ArgMax) {
@ -983,7 +983,7 @@ TEST(ReductionOpTest, ArgMax) {
{1, 1,
1, 1,
1, 1});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0
}
TEST(ReductionOpTest, ArgMax_do_not_keepdims) {
@ -1003,7 +1003,7 @@ TEST(ReductionOpTest, ArgMax_do_not_keepdims) {
{1, 1,
1, 1,
1, 1});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0
}
TEST(ReductionOpTest, ArgMax_do_not_keepdims_2) {
@ -1014,7 +1014,7 @@ TEST(ReductionOpTest, ArgMax_do_not_keepdims_2) {
{1.0f, 2.0f, 3.0f});
test.AddOutput<int64_t>("reduced", {},
{2});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ArgMax_int32) {
@ -1034,7 +1034,7 @@ TEST(ReductionOpTest, ArgMax_int32) {
{1, 1,
1, 1,
1, 1});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ArgMax2D) {
@ -1047,7 +1047,7 @@ TEST(ReductionOpTest, ArgMax2D) {
9.0f, 10.0f});
test.AddOutput<int64_t>("reduced", {3, 1},
{1, 0, 1});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0
}
TEST(ReductionOpTest, ArgMin) {
@ -1066,7 +1066,7 @@ TEST(ReductionOpTest, ArgMin) {
test.AddOutput<int64_t>("reduced", {1, 2, 2},
{0, 0,
0, 0});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ArgMin_do_not_keepdims) {
@ -1085,7 +1085,7 @@ TEST(ReductionOpTest, ArgMin_do_not_keepdims) {
test.AddOutput<int64_t>("reduced", {2, 2},
{0, 0,
0, 0});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ArgMin_do_not_keepdims_2) {
@ -1095,7 +1095,7 @@ TEST(ReductionOpTest, ArgMin_do_not_keepdims_2) {
test.AddInput<float>("data", {3},
{1.0f, 2.0f, 3.0f});
test.AddOutput<int64_t>("reduced", {}, {0});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(ReductionOpTest, ArgMin_int32) {
@ -1114,7 +1114,7 @@ TEST(ReductionOpTest, ArgMin_int32) {
test.AddOutput<int64_t>("reduced", {2, 2},
{0, 0,
0, 0});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
} // namespace test

23
onnxruntime/test/providers/cpu/tensor/concat_op_test.cc
View file

@ -7,7 +7,8 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on some of the tests because the limit in its parser: axis >=0 && axis < nbDims
// Some of the tests can't run on TensorrtExecutionProvider because of unsupported data types or limits
// in its parser: axis >=0 && axis < nbDims. Those Tests will fallback to other EPs
TEST(MathOpTest, Concat1D_string) {
OpTester test("Concat");
@ -17,7 +18,7 @@ TEST(MathOpTest, Concat1D_string) {
test.AddInput<std::string>("input2", {2}, {"2", "3"});
test.AddInput<std::string>("input3", {4}, {"4", "5", "6", "7"});
test.AddOutput<std::string>("concat_result", {7}, {"1", "2", "3", "4", "5", "6", "7"});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat1D_int32) {
@ -28,7 +29,7 @@ TEST(MathOpTest, Concat1D_int32) {
test.AddInput<int32_t>("input2", {2}, {2, 3});
test.AddInput<int32_t>("input3", {4}, {4, 5, 6, 7});
test.AddOutput<int32_t>("concat_result", {7}, {1, 2, 3, 4, 5, 6, 7});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat1D_int32_negative_axis) {
@ -39,7 +40,7 @@ TEST(MathOpTest, Concat1D_int32_negative_axis) {
test.AddInput<int32_t>("input2", {2}, {2, 3});
test.AddInput<int32_t>("input3", {4}, {4, 5, 6, 7});
test.AddOutput<int32_t>("concat_result", {7}, {1, 2, 3, 4, 5, 6, 7});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat1D_1) {
@ -50,7 +51,7 @@ TEST(MathOpTest, Concat1D_1) {
test.AddInput<float>("input2", {2}, {2.0f, 3.0f});
test.AddInput<float>("input3", {4}, {4.0f, 5.0f, 6.0f, 7.0f});
test.AddOutput<float>("concat_result", {7}, {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat1D_2) {
@ -61,7 +62,7 @@ TEST(MathOpTest, Concat1D_2) {
test.AddInput<float>("input2", {2}, {2.0f, 3.0f});
test.AddInput<float>("input3", {0}, {});
test.AddOutput<float>("concat_result", {3}, {1.0f, 2.0f, 3.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat2D_1) {
@ -76,7 +77,7 @@ TEST(MathOpTest, Concat2D_1) {
{11.0f, 12.0f, 13.0f, 14.0f,
21.0f, 22.0f, 23.0f, 24.0f,
31.0f, 32.0f, 33.0f, 34.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat2D_2) {
@ -92,7 +93,7 @@ TEST(MathOpTest, Concat2D_2) {
21.0f, 22.0f, 23.0f, 24.0f,
31.0f, 32.0f, 33.0f, 34.0f,
41.0f, 42.0f, 43.0f, 44.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat2D_3) {
@ -103,7 +104,7 @@ TEST(MathOpTest, Concat2D_3) {
test.AddInput<float>("input2", {1, 0}, {});
test.AddInput<float>("input3", {1, 0}, {});
test.AddOutput<float>("concat_result", {1, 0}, {});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat3D_1) {
@ -135,7 +136,7 @@ TEST(MathOpTest, Concat3D_1) {
311.0f, 312.0f, 313.0f,
321.0f, 322.0f, 323.0f,
331.0f, 332.0f, 333.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat3D_1_negative_axis) {
@ -167,7 +168,7 @@ TEST(MathOpTest, Concat3D_1_negative_axis) {
311.0f, 312.0f, 313.0f,
321.0f, 322.0f, 323.0f,
331.0f, 332.0f, 333.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(MathOpTest, Concat3D_2) {

31
onnxruntime/test/providers/cpu/tensor/gather_op_test.cc
View file

@ -7,7 +7,8 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on some of the tests because of unsupported data types
// Some of the tests can't run on TensorrtExecutionProvider because of unsupported data types.
// Those tests will fallback to other EPs
TEST(GatherOpTest, Gather_axis0) {
OpTester test("Gather");
@ -24,7 +25,7 @@ TEST(GatherOpTest, Gather_axis0) {
{10.0f, 10.1f, 10.2f, 10.3f,
11.0f, 11.1f, 11.2f, 11.3f,
12.0f, 12.1f, 12.2f, 12.3f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_negative_axis) {
@ -42,7 +43,7 @@ TEST(GatherOpTest, Gather_negative_axis) {
{10.0f, 10.1f, 10.2f, 10.3f,
11.0f, 11.1f, 11.2f, 11.3f,
12.0f, 12.1f, 12.2f, 12.3f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_invalid_axis) {
@ -114,7 +115,7 @@ TEST(GatherOpTest, Gather_axis1) {
0.0f, 0.1f, 0.2f, 0.3f,
12.0f, 12.1f, 12.2f, 12.3f,
10.0f, 10.1f, 10.2f, 10.3f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis2) {
@ -135,7 +136,7 @@ TEST(GatherOpTest, Gather_axis2) {
10.1f, 10.0f, 10.2f,
11.1f, 11.0f, 11.2f,
12.1f, 12.0f, 12.2f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis0_indices2d) {
@ -151,7 +152,7 @@ TEST(GatherOpTest, Gather_axis0_indices2d) {
test.AddOutput<float>("output", {2, 2, 3},
{1.0f, 1.1f, 1.2f, 0.0f, 0.1f, 0.2f,
2.0f, 2.1f, 2.2f, 1.0f, 1.1f, 1.2f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d) {
@ -168,7 +169,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d) {
{0.1f, 0.0f, 0.2f, 0.1f,
1.1f, 1.0f, 1.2f, 1.1f,
2.1f, 2.0f, 2.2f, 2.1f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d_int32) {
@ -185,7 +186,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_int32) {
{1, 0, 2, 1,
11, 10, 12, 11,
21, 20, 22, 21});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Input batch size is inconsistent
}
TEST(GatherOpTest, Gather_axis1_indices2d_uint32) {
@ -202,7 +203,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_uint32) {
{1, 0, 2, 1,
11, 10, 12, 11,
21, 20, 22, 21});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d_int16) {
@ -219,7 +220,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_int16) {
{1, 0, 2, 1,
11, 10, 12, 11,
21, 20, 22, 21});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d_uint16) {
@ -236,7 +237,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_uint16) {
{1, 0, 2, 1,
11, 10, 12, 11,
21, 20, 22, 21});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d_int8) {
@ -253,7 +254,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_int8) {
{1, 0, 2, 1,
11, 10, 12, 11,
21, 20, 22, 21});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d_string) {
@ -270,7 +271,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_string) {
{"1", "0", "2", "1",
"11", "10", "12", "11",
"21", "20", "22", "21"});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_axis1_indices2d_bool) {
@ -287,7 +288,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_bool) {
{false, true, true, false,
true, true, false, true,
true, false, false, true});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(GatherOpTest, Gather_perf) {
@ -302,7 +303,7 @@ TEST(GatherOpTest, Gather_perf) {
test.AddInput<int32_t>("data", {50000, 100}, input);
test.AddInput<int32_t>("indices", {800, 1}, indices);
test.AddOutput<int32_t>("output", {800, 1, 100}, output);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
} // namespace test
} // namespace onnxruntime

21
onnxruntime/test/providers/cpu/tensor/pad_test.cc
View file

@ -7,7 +7,8 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on some of the tests because only constant mode and value 0 is supported for "Pad" node
// Some of the tests can't run on TensorrtExecutionProvider because only constant mode and value 0 of "Pad" node is supported.
// Those tests will fallback to other EP.
TEST(TensorOpTest, Pad_Spec_Example) {
OpTester test("Pad");
@ -16,7 +17,7 @@ TEST(TensorOpTest, Pad_Spec_Example) {
test.AddAttribute("value", 0.0f);
test.AddInput<float>("data", {3, 2}, {1.0f, 1.2f, 2.3f, 3.4f, 4.5f, 5.7f});
test.AddOutput<float>("output", {3, 4}, {0.0f, 0.0f, 1.0f, 1.2f, 0.0f, 0.0f, 2.3f, 3.4f, 0.0f, 0.0f, 4.5f, 5.7f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Constant_1D) {
@ -26,7 +27,7 @@ TEST(TensorOpTest, Pad_Constant_1D) {
test.AddAttribute("value", 1234.0f);
test.AddInput<float>("data", {2}, {1.0f, 2.0f});
test.AddOutput<float>("output", {5}, {1234.0f, 1.0f, 2.0f, 1234.0f, 1234.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Constant_1D_Zero) {
@ -36,7 +37,7 @@ TEST(TensorOpTest, Pad_Constant_1D_Zero) {
test.AddAttribute("value", 1234.0f);
test.AddInput<float>("data", {2}, {1.0f, 2.0f});
test.AddOutput<float>("output", {2}, {1.0f, 2.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Constant_2D) {
@ -52,7 +53,7 @@ TEST(TensorOpTest, Pad_Constant_2D) {
1234.0f, 1234.0f, 11.0f, 21.0f, 1234.0f, 1234.0f,
1234.0f, 1234.0f, 12.0f, 22.0f, 1234.0f, 1234.0f,
1234.0f, 1234.0f, 1234.0f, 1234.0f, 1234.0f, 1234.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Constant_2D_negative) {
@ -68,7 +69,7 @@ TEST(TensorOpTest, Pad_Constant_2D_negative) {
1234.0f, 1234.0f, 11.0f, 21.0f,
1234.0f, 1234.0f, 12.0f, 22.0f,
1234.0f, 1234.0f, 1234.0f, 1234.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_3D_complex) {
@ -88,7 +89,7 @@ TEST(TensorOpTest, Pad_3D_complex) {
111.0f, 112.0f,
121.0f, 122.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Edge_2D) {
@ -106,7 +107,7 @@ TEST(TensorOpTest, Pad_Edge_2D) {
12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f,
12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f,
12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Edge_3D) {
@ -139,7 +140,7 @@ TEST(TensorOpTest, Pad_Edge_3D) {
12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f,
12.0f, 12.0f, 12.0f, 22.0f, 32.0f, 32.0f, 32.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Pad_Reflect_2D) {
@ -159,7 +160,7 @@ TEST(TensorOpTest, Pad_Reflect_2D) {
33.0f, 23.0f, 13.0f, 23.0f, 33.0f, 23.0f, 13.0f,
32.0f, 22.0f, 12.0f, 22.0f, 32.0f, 22.0f, 12.0f,
31.0f, 21.0f, 11.0f, 21.0f, 31.0f, 21.0f, 11.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
} // namespace test

15
onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc
View file

@ -7,14 +7,15 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on the tests because axis=0 is not supported
// Some of the tests can't run on TensorrtExecutionProvider because axis=0 is not supported
// or there are unsupported data types. Those tests will fallback to other EPs.
TEST(SqueezeOpTest, Squeeze_1) {
OpTester test("Squeeze");
test.AddAttribute("axes", std::vector<int64_t>{0});
test.AddInput<float>("data", {1, 3, 4, 5}, std::vector<float>(60, 1.0f));
test.AddOutput<float>("squeezed", {3, 4, 5}, std::vector<float>(60, 1.0f));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(SqueezeOpTest, Squeeze_1_int32) {
@ -22,7 +23,7 @@ TEST(SqueezeOpTest, Squeeze_1_int32) {
test.AddAttribute("axes", std::vector<int64_t>{0});
test.AddInput<int32_t>("data", {1, 3, 4, 5}, std::vector<int32_t>(60, 1));
test.AddOutput<int32_t>("squeezed", {3, 4, 5}, std::vector<int32_t>(60, 1));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(SqueezeOpTest, Squeeze_string) {
@ -30,7 +31,7 @@ TEST(SqueezeOpTest, Squeeze_string) {
test.AddAttribute("axes", std::vector<int64_t>{0, 2, 4});
test.AddInput<std::string>("data", {1, 2, 1, 3, 1}, std::vector<std::string>({"1", "2", "3", "4", "5", "6"}));
test.AddOutput<std::string>("squeezed", {2, 3}, std::vector<std::string>({"1", "2", "3", "4", "5", "6"}));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(SqueezeOpTest, Squeeze_2) {
@ -40,7 +41,7 @@ TEST(SqueezeOpTest, Squeeze_2) {
std::vector<float>{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f});
test.AddOutput<float>("squeezed", {4, 2},
std::vector<float>{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(SqueezeOpTest, UnsortedAxes) {
@ -51,7 +52,7 @@ TEST(SqueezeOpTest, UnsortedAxes) {
std::vector<float>{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f});
test.AddOutput<float>("squeezed", {4, 2},
std::vector<float>{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(SqueezeOpTest, DuplicateAxes) {
@ -62,7 +63,7 @@ TEST(SqueezeOpTest, DuplicateAxes) {
std::vector<float>{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f});
test.AddOutput<float>("squeezed", {4, 2},
std::vector<float>{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(SqueezeOpTest, BadAxes) {

15
onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc
View file

@ -12,7 +12,8 @@ namespace test {
using ExpectResult = OpTester::ExpectResult;
// Disable TensorRT on some of the tests because of unsupported data types
// Some of the tests can't run on TensorrtExecutionProvider because of unsupported data types.
// Those tests will fallback to other EPs.
TEST(TensorOpTest, Reshape) {
OpTester test("Reshape");
@ -20,7 +21,7 @@ TEST(TensorOpTest, Reshape) {
test.AddInput<float>("data", {2, 3}, std::vector<float>(6, 1.0f));
test.AddInput<int64_t>("shape", {3}, {-1, 0, 2});
test.AddOutput<float>("reshaped", {1, 3, 2}, std::vector<float>(6, 1.0f));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNupharExecutionProvider, kTensorrtExecutionProvider}); // Nuphar only supports reshape shape from initializer
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNupharExecutionProvider}); // Nuphar only supports reshape shape from initializer
}
TEST(TensorOpTest, ReshapeWithEmptyDim) {
@ -29,7 +30,7 @@ TEST(TensorOpTest, ReshapeWithEmptyDim) {
test.AddInput<float>("data", {1, 1, 1}, std::vector<float>(1, 1.0f));
test.AddInput<int64_t>("shape", {0}, {}, true);
test.AddOutput<float>("reshaped", {}, std::vector<float>(1, 1.0f));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, ReshapeWithInitializer) {
@ -38,7 +39,7 @@ TEST(TensorOpTest, ReshapeWithInitializer) {
test.AddInput<float>("data", {2, 3}, std::vector<float>(6, 1.0f));
test.AddInput<int64_t>("shape", {3}, {-1, 0, 2}, true);
test.AddOutput<float>("reshaped", {1, 3, 2}, std::vector<float>(6, 1.0f));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Identity) {
@ -54,7 +55,7 @@ TEST(TensorOpTest, IdentityString) {
std::vector<std::string> X{"this", "is", "a", "test", "for", "identity"};
test.AddInput<std::string>("input", {2, 3}, X);
test.AddOutput<std::string>("output", {2, 3}, X);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, ShapeTest2D) {
@ -62,7 +63,7 @@ TEST(TensorOpTest, ShapeTest2D) {
test.AddInput<float>("data", {2, 3}, std::vector<float>(6, 1.0f));
test.AddOutput<int64_t>("shape", {2}, {2, 3});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: volume of dimensions is not consistent with weights size
}
TEST(TensorOpTest, ShapeTest3D) {
@ -70,7 +71,7 @@ TEST(TensorOpTest, ShapeTest3D) {
test.AddInput<float>("data", {2, 3, 4}, std::vector<float>(24, 1.0f));
test.AddOutput<int64_t>("shape", {3}, {2, 3, 4});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: volume of dimensions is not consistent with weights size
}
template <typename SrcType,

17
onnxruntime/test/providers/cpu/tensor/transpose_test.cc
View file

@ -7,20 +7,27 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on the tests because of errors in the parser
// Some of the tests can't run on TensorrtExecutionProvider because of errors.
// Those tests will fallback to other EPs.
template <class T>
void TransposeTest(std::vector<int64_t>& input_shape,
std::vector<T>& input_vals,
std::vector<int64_t>* p_perm,
std::vector<int64_t> expected_shape,
std::initializer_list<T>& expected_vals) {
std::initializer_list<T>& expected_vals,
bool is_tensorrt_supported = true) {
OpTester test("Transpose");
if (nullptr != p_perm)
test.AddAttribute("perm", *p_perm);
test.AddInput<T>("X", input_shape, input_vals);
test.AddOutput<T>("Y", expected_shape, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
// Disable TensorRT on unsupported tests
std::unordered_set<std::string> excluded_providers;
if (!is_tensorrt_supported) {
excluded_providers.insert(kTensorrtExecutionProvider);
}
test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_providers);
}
// Test 2 dimensional transpose, with no permutation attribute specified
@ -36,7 +43,7 @@ TEST(TransposeOpTest, TwoDimNoAttr) {
2.0f, 5.0f,
3.0f, 6.0f};
TransposeTest(input_shape, input_vals, nullptr, expected_shape, expected_vals);
TransposeTest(input_shape, input_vals, nullptr, expected_shape, expected_vals, false);//TensorRT: SegFault error
}
TEST(TransposeOpTest, TwoDimNoAttrStr) {
@ -136,7 +143,7 @@ TEST(TransposeOpTest, ThreeDim) {
};
TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals);
TransposeTest(input_shape, input_vals, &perm, expected_shape, expected_vals, false); //TensorRT: illegal error
}
TEST(TransposeOpTest, ThreeDimStr) {

10
onnxruntime/test/providers/cpu/tensor/unsqueeze_op_test.cc
View file

@ -7,7 +7,7 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on the tests because of errors in the parser
// Disable TensorRT on the tests because of SegFault errors in the parser
TEST(TensorOpTest, Unsqueeze_1) {
OpTester test("Unsqueeze");
@ -33,7 +33,7 @@ TEST(TensorOpTest, Unsqueeze_2) {
test.AddAttribute("axes", std::vector<int64_t>{0, 4});
test.AddInput<float>("input", {2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.AddOutput<float>("output", {1, 2, 3, 4, 1}, std::vector<float>(2 * 3 * 4, 1.0f));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Unsqueeze_3) {
@ -42,7 +42,7 @@ TEST(TensorOpTest, Unsqueeze_3) {
test.AddAttribute("axes", std::vector<int64_t>{2, 1, 0});
test.AddInput<float>("input", {2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.AddOutput<float>("output", {1, 1, 1, 2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run();
}
TEST(TensorOpTest, Unsqueeze_Duplicate) {
@ -51,7 +51,7 @@ TEST(TensorOpTest, Unsqueeze_Duplicate) {
test.AddAttribute("axes", std::vector<int64_t>{2, 1, 0, 2});
test.AddInput<float>("input", {2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.AddOutput<float>("output", {1, 1, 1, 2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.Run(OpTester::ExpectResult::kExpectFailure, "'axes' has a duplicate axis", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectFailure, "'axes' has a duplicate axis");
}
TEST(TensorOpTest, Unsqueeze_OutOfRange) {
@ -60,7 +60,7 @@ TEST(TensorOpTest, Unsqueeze_OutOfRange) {
test.AddAttribute("axes", std::vector<int64_t>{4});
test.AddInput<float>("input", {2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.AddOutput<float>("output", {2, 1, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
test.Run(OpTester::ExpectResult::kExpectFailure, "Mismatch between number of source and target dimensions.", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectFailure, "Mismatch between number of source and target dimensions.");
}
} // namespace test

21
onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc
View file

@ -8,7 +8,8 @@
namespace onnxruntime {
namespace test {
// Disable TensorRT on some of the tests because TensorRT only supports "nearest" mode Upsample and limited data types
// Some of the tests can't run on TensorrtExecutionProvider because TensorRT only supports "nearest" mode Upsample
// and limited data types. Those tests will fallback to other EPs
TEST(UpsampleOpTest, UpsampleOpNearestTest) {
OpTester test("Upsample");
@ -69,7 +70,7 @@ TEST(UpsampleOpTest, UpsampleOpNearestTest_int32) {
7, 7, 7, 9, 9, 9};
test.AddOutput<int32_t>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: nvinfer1::query::Ports<nvinfer1::query::AbstractTensor>&): Assertion `!formats.empty()' failed
}
TEST(UpsampleOpTest, UpsampleOpNearestTest_uint8) {
@ -100,7 +101,7 @@ TEST(UpsampleOpTest, UpsampleOpNearestTest_uint8) {
7, 7, 7, 9, 9, 9};
test.AddOutput<uint8_t>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: unsupported data type
test.Run();
}
TEST(UpsampleOpTest, UpsampleOpNearest2XTest) {
@ -173,7 +174,7 @@ TEST(UpsampleOpTest, UpsampleOpNearest222XTest) {
};
test.AddOutput<float>("Y", {N*2, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT parser: Assertion failed: scales[0] == 1 && scales[1] == 1
test.Run();
}
TEST(UpsampleOpTest, UpsampleOpNearest15XTest) {
@ -235,7 +236,7 @@ TEST(UpsampleOpTest, UpsampleOpNearest2XTest_int32) {
7, 7, 9, 9};
test.AddOutput<int32_t>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: nvinfer1::query::Ports<nvinfer1::query::AbstractTensor>&): Assertion `!formats.empty()' failed
}
TEST(UpsampleOpTest, UpsampleOpBilinearTest) {
@ -266,7 +267,7 @@ TEST(UpsampleOpTest, UpsampleOpBilinearTest) {
7.0f, 7.5f, 8.0f, 8.5f, 9.0f, 9.0f, 9.0f, 9.0f};
test.AddOutput<float>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT parser:Assertion failed: mode == "nearest"
test.Run();
}
TEST(UpsampleOpTest, UpsampleOpBilinearTest2) {
@ -297,7 +298,7 @@ TEST(UpsampleOpTest, UpsampleOpBilinearTest2) {
7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 11.0f, 11.0f, 11.0f};
test.AddOutput<float>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT parser:Assertion failed: mode == "nearest"
test.Run();
}
TEST(UpsampleOpTest, UpsampleOpBilinearTest_int32) {
@ -328,7 +329,7 @@ TEST(UpsampleOpTest, UpsampleOpBilinearTest_int32) {
7, 7, 8, 8, 9, 9, 9, 9};
test.AddOutput<int32_t>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT parser:Assertion failed: mode == "nearest"
test.Run();
}
TEST(UpsampleOpTest, UpsampleOpNearestTest_1D) {
@ -350,7 +351,7 @@ TEST(UpsampleOpTest, UpsampleOpNearestTest_1D) {
5.0f, 5.0f};
test.AddOutput<float>("Y", {10}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT parser:Assertion failed: tensor.getDimensions().nbDims == 3
test.Run();
}
TEST(UpsampleOpTest, UpsampleOpNearest2XTest_opset9) {
@ -381,7 +382,7 @@ TEST(UpsampleOpTest, UpsampleOpNearest2XTest_opset9) {
7, 7, 9, 9};
test.AddOutput<int32_t>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT parser:Assertion failed: scales_input.is_weights()
test.Run();
}
} // namespace test
} // namespace onnxruntime

5
tools/ci_build/build.py
View file

@ -495,8 +495,11 @@ def setup_tensorrt_vars(args):
# Set maximum batch size for TensorRT. The number needs to be no less than maximum batch size in all unit tests
os.environ["ORT_TENSORRT_MAX_BATCH_SIZE"] = "13"
# Set maximum workspace size in byte for TensorRT (1GB = 1073741824 bytes).
# Set maximum workspace size in byte for TensorRT (1GB = 1073741824 bytes)
os.environ["ORT_TENSORRT_MAX_WORKSPACE_SIZE"] = "1073741824"
# Set maximum number of iterations to detect unsupported nodes and partition the models for TensorRT
os.environ["ORT_TENSORRT_MAX_PARSER_ITERATIONS"] = "6"
return tensorrt_home