mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-27 20:02:15 +00:00
Handle compiler warnings for TRT EP (#9956)
* fix error C4996 * remove wd4996 and fix error C4966 * fix typo * remove wd4996 for onnx-tensorrt * remove more /wd for onnx-tensorrt * gix bug for strncpy_s of (Buffer is too small && 0) * fix code to remove warning 4244 * fix code to remove warning 4267 * remove /wd4267 /wd4244 * fix bug * change int to size_t * using size_t instead of int * use float instead of double * Use size_t instead of int * use size_t instead of int * use size_t instead of int. Also fix typo
This commit is contained in:
Chi Lo
GitHub
parent
e0960d7d79
commit
4669048b47
4 changed files with 93 additions and 74 deletions
|
|
@ -499,13 +499,13 @@ if (onnxruntime_USE_TENSORRT)
|
|||
if (WIN32)
|
||||
add_definitions(-D_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING=1)
|
||||
set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd4324 /wd4701 /wd4804 /wd4702 /wd4458 /wd4703")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd4324 /wd4701 /wd4804 /wd4702 /wd4458 /wd4703")
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4805")
|
||||
endif()
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -include algorithm")
|
||||
set(PROTOBUF_LIBRARY libprotobuf)
|
||||
set(DISABLED_WARNINGS_FOR_TRT /wd4267 /wd4244 /wd4996 /wd4456)
|
||||
set(DISABLED_WARNINGS_FOR_TRT /wd4456)
|
||||
endif()
|
||||
if ( CMAKE_COMPILER_IS_GNUCC )
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-missing-field-initializers")
|
||||
|
|
@ -550,7 +550,7 @@ if (onnxruntime_USE_TENSORRT)
|
|||
target_compile_definitions(onnxruntime_providers_tensorrt PRIVATE ONNXIFI_BUILD_LIBRARY=1)
|
||||
target_compile_options(onnxruntime_providers_tensorrt PRIVATE ${DISABLED_WARNINGS_FOR_TRT})
|
||||
if (WIN32)
|
||||
target_compile_options(onnxruntime_providers_tensorrt INTERFACE /wd4267 /wd4244 /wd4996 /wd4456)
|
||||
target_compile_options(onnxruntime_providers_tensorrt INTERFACE /wd4456)
|
||||
endif()
|
||||
|
||||
# Needed for the provider interface, as it includes training headers when training is enabled
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ float ConvertSinglePrecisionIEEE754ToFloat(unsigned long input) {
|
|||
for (int i = 0; i < 23; ++i) {
|
||||
m += ((input >> (23 - i - 1)) & 0x01) * pow(2.0, p--);
|
||||
}
|
||||
return (s ? -1 : 1) * pow(2.0, e) * (m + 1.0);
|
||||
return static_cast<float>((s ? -1 : 1) * pow(2.0, e) * (m + 1.0));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -72,7 +72,7 @@ float ConvertSinglePrecisionIEEE754ToFloat(unsigned long input) {
|
|||
* key: tensor_a, value: dim_0 min_shape max_shape dim_2 min_shape max_shape
|
||||
* key: tensor_b, value: dim_1 min_shape max_shape
|
||||
*/
|
||||
void SerializeProfile(const std::string& file_name, std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64_t>>>& shape_ranges) {
|
||||
void SerializeProfile(const std::string& file_name, std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>>& shape_ranges) {
|
||||
// Serialize profile
|
||||
flexbuffers::Builder builder;
|
||||
auto profile_start = builder.StartMap();
|
||||
|
|
@ -97,7 +97,7 @@ void SerializeProfile(const std::string& file_name, std::unordered_map<std::stri
|
|||
}
|
||||
|
||||
// Deserialize engine profile
|
||||
std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64_t>>> DeserializeProfile(std::ifstream& infile) {
|
||||
std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> DeserializeProfile(std::ifstream& infile) {
|
||||
// Load flexbuffer
|
||||
infile.seekg(0, std::ios::end);
|
||||
size_t length = infile.tellg();
|
||||
|
|
@ -107,13 +107,13 @@ std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64
|
|||
infile.close();
|
||||
|
||||
// Deserialize profile
|
||||
std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64_t>>> shape_ranges;
|
||||
std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> shape_ranges;
|
||||
auto tensors_range_entries = flexbuffers::GetRoot((const uint8_t*)data.get(), length).AsMap();
|
||||
auto keys = tensors_range_entries.Keys();
|
||||
auto values = tensors_range_entries.Values();
|
||||
for (size_t i = 0, end = keys.size(); i < end; ++i) {
|
||||
auto dim_range_vectors = values[i].AsTypedVector();
|
||||
std::unordered_map<int, std::pair<int64_t, int64_t>> inner_map;
|
||||
std::unordered_map<size_t, std::pair<int64_t, int64_t>> inner_map;
|
||||
for (size_t j = 0, end = dim_range_vectors.size() / 3; j < end; ++j) {
|
||||
size_t idx = 3 * j;
|
||||
inner_map[dim_range_vectors[idx].AsInt64()] = std::make_pair(dim_range_vectors[idx + 1].AsInt64(), dim_range_vectors[idx + 2].AsInt64());
|
||||
|
|
@ -124,7 +124,7 @@ std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64
|
|||
}
|
||||
|
||||
// Check if cycle exists in the graph after partitioning
|
||||
bool FindCycleHelper(int i, const std::list<int>* adjacency_map, bool visited[], bool* st, std::vector<int>& cycles) {
|
||||
bool FindCycleHelper(size_t i, const std::list<size_t>* adjacency_map, bool visited[], bool* st, std::vector<size_t>& cycles) {
|
||||
if (!visited[i]) {
|
||||
visited[i] = true;
|
||||
st[i] = true;
|
||||
|
|
@ -184,7 +184,7 @@ bool ReadDynamicRange(const std::string file_name, const bool is_trt_calibration
|
|||
std::getline(in_line, str, delim);
|
||||
unsigned long scale_int = std::strtoul(str.c_str(), nullptr, 16);
|
||||
float scale_float = ConvertSinglePrecisionIEEE754ToFloat(scale_int);
|
||||
float dynamic_range = scale_float * 127.0;
|
||||
float dynamic_range = scale_float * 127.0f;
|
||||
dynamic_range_map[tensor_name] = dynamic_range;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -202,7 +202,8 @@ bool ReadDynamicRange(const std::string file_name, const bool is_trt_calibration
|
|||
auto flat_table = flatbuffers::GetRoot<CalTableFlatBuffers::TrtTable>((const uint8_t*)data.get());
|
||||
auto flat_dict = flat_table->dict();
|
||||
for (size_t i = 0, end = flat_dict->size(); i < end; ++i) {
|
||||
dynamic_range_map[flat_dict->Get(i)->key()->str()] = std::stof(flat_dict->Get(i)->value()->str());
|
||||
flatbuffers::uoffset_t idx = static_cast<flatbuffers::uoffset_t>(i);
|
||||
dynamic_range_map[flat_dict->Get(idx)->key()->str()] = std::stof(flat_dict->Get(idx)->value()->str());
|
||||
}
|
||||
}
|
||||
return true;
|
||||
|
|
@ -255,7 +256,7 @@ bool SetDynamicRange(nvinfer1::INetworkDefinition& network, std::unordered_map<s
|
|||
}
|
||||
max_weight = std::max(max_weight, std::abs(weight));
|
||||
}
|
||||
if (!trt_layer->getOutput(j)->setDynamicRange(-max_weight, max_weight)) {
|
||||
if (!trt_layer->getOutput(j)->setDynamicRange(static_cast<float>(-max_weight), static_cast<float>(max_weight))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -959,8 +960,8 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
|
|||
SubGraphCollection_t next_nodes_list;
|
||||
const std::vector<NodeIndex>& subgraph_node_index = graph_viewer->GetNodesInTopologicalOrder();
|
||||
next_nodes_list = GetSupportedList(parser_nodes_list, iterations, max_iterations, *graph_viewer, early_termination);
|
||||
for (int i = 0, end = next_nodes_list.size(); i < end; ++i) {
|
||||
for (int j = 0, end = next_nodes_list[i].first.size(); j < end; ++j) {
|
||||
for (size_t i = 0, end = next_nodes_list.size(); i < end; ++i) {
|
||||
for (size_t j = 0, end = next_nodes_list[i].first.size(); j < end; ++j) {
|
||||
next_nodes_list[i].first[j] = group.first[subgraph_node_index[next_nodes_list[i].first[j]]];
|
||||
}
|
||||
nodes_list_output.push_back(next_nodes_list[i]);
|
||||
|
|
@ -977,11 +978,11 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t&
|
|||
bool trt_cycle = true;
|
||||
while (trt_cycle) {
|
||||
trt_cycle = false;
|
||||
std::unordered_map<std::string, int> node_to_index_map;
|
||||
std::unordered_map<int, std::string> index_to_node_map;
|
||||
std::unordered_map<std::string, size_t> node_to_index_map;
|
||||
std::unordered_map<size_t, std::string> index_to_node_map;
|
||||
std::unordered_map<std::string, std::unordered_set<std::string>> input_to_nodes_map, node_to_outputs_map;
|
||||
std::unordered_set<int> non_trt_node_index(node_index.begin(), node_index.end());
|
||||
int id = 0;
|
||||
std::unordered_set<size_t> non_trt_node_index(node_index.begin(), node_index.end());
|
||||
size_t id = 0;
|
||||
for (const auto& group : supported_nodes_vector) {
|
||||
if (!group.first.empty()) {
|
||||
// Construct subgraph from node list
|
||||
|
|
@ -1030,15 +1031,15 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t&
|
|||
}
|
||||
|
||||
// Create adjacency list
|
||||
int graph_size = node_to_index_map.size();
|
||||
std::list<int>* adjacency_map = new std::list<int>[graph_size];
|
||||
size_t graph_size = node_to_index_map.size();
|
||||
std::list<size_t>* adjacency_map = new std::list<size_t>[graph_size];
|
||||
for (const auto& node : node_to_outputs_map) {
|
||||
for (auto iter = node.second.begin(); iter != node.second.end(); ++iter) {
|
||||
const auto& loc = input_to_nodes_map.find(*iter);
|
||||
if (loc != input_to_nodes_map.end()) {
|
||||
int parent_node_index = node_to_index_map.find(node.first)->second;
|
||||
size_t parent_node_index = node_to_index_map.find(node.first)->second;
|
||||
for (auto child_node : loc->second) {
|
||||
int child_node_index = node_to_index_map.find(child_node)->second;
|
||||
size_t child_node_index = node_to_index_map.find(child_node)->second;
|
||||
adjacency_map[parent_node_index].push_back(child_node_index);
|
||||
}
|
||||
}
|
||||
|
|
@ -1048,14 +1049,14 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t&
|
|||
// Check cycle in the graph
|
||||
bool* visited = new bool[graph_size];
|
||||
bool* st = new bool[graph_size];
|
||||
for (int i = 0; i < graph_size; ++i) {
|
||||
for (size_t i = 0; i < graph_size; ++i) {
|
||||
visited[i] = false;
|
||||
st[i] = false;
|
||||
}
|
||||
|
||||
std::vector<int> cycles;
|
||||
std::vector<size_t> cycles;
|
||||
bool has_cycle = false;
|
||||
for (int i = 0; i < graph_size; ++i) {
|
||||
for (size_t i = 0; i < graph_size; ++i) {
|
||||
if (FindCycleHelper(i, adjacency_map, visited, st, cycles)) {
|
||||
has_cycle = true;
|
||||
break;
|
||||
|
|
@ -1064,7 +1065,7 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t&
|
|||
|
||||
// Remove TensorRT subgraph from the supported node list if it's part of the cycle
|
||||
if (has_cycle) {
|
||||
for (int i = 0; i < static_cast<int>(cycles.size()); ++i) {
|
||||
for (size_t i = 0; i < cycles.size(); ++i) {
|
||||
auto loc = index_to_node_map.find(cycles[i]);
|
||||
if (loc != index_to_node_map.end() && loc->second.find("TRTKernel") != std::string::npos) {
|
||||
supported_nodes_vector.erase(supported_nodes_vector.begin() + cycles[i]);
|
||||
|
|
@ -1086,7 +1087,7 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
|
|||
// Get ModelPath
|
||||
const auto& path_string = graph.ModelPath().ToPathString();
|
||||
#ifdef _WIN32
|
||||
wcstombs(model_path_, path_string.c_str(), sizeof(model_path_));
|
||||
wcstombs_s(nullptr, model_path_, sizeof(model_path_), path_string.c_str(), sizeof(model_path_));
|
||||
#else
|
||||
strcpy(model_path_, path_string.c_str());
|
||||
#endif
|
||||
|
|
@ -1104,7 +1105,7 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
|
|||
|
||||
// Remove subgraphs if its size is less than the predefined minimal size
|
||||
for (auto it = supported_nodes_vector.begin(); it != supported_nodes_vector.end(); ++it) {
|
||||
const int subgraph_size = it->first.size();
|
||||
const size_t subgraph_size = it->first.size();
|
||||
if (subgraph_size < min_subgraph_size_) {
|
||||
supported_nodes_vector.erase(it--);
|
||||
}
|
||||
|
|
@ -1120,17 +1121,17 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
|
|||
if (!group.first.empty()) {
|
||||
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(group, graph);
|
||||
result.push_back(ComputeCapability::Create(std::move(sub_graph)));
|
||||
number_of_trt_nodes += group.first.size();
|
||||
number_of_trt_nodes += static_cast<int>(group.first.size());
|
||||
}
|
||||
}
|
||||
|
||||
const int number_of_subgraphs = supported_nodes_vector.size();
|
||||
const size_t number_of_subgraphs = supported_nodes_vector.size();
|
||||
if (number_of_trt_nodes == 0) {
|
||||
LOGS_DEFAULT(WARNING) << "[TensorRT EP] No graph will run on TensorRT exeuction provider";
|
||||
LOGS_DEFAULT(WARNING) << "[TensorRT EP] No graph will run on TensorRT execution provider";
|
||||
} else if (number_of_trt_nodes == number_of_ort_nodes) {
|
||||
LOGS_DEFAULT(INFO) << "[TensorRT EP] Whole graph will run on TensorRT exeuction provider";
|
||||
LOGS_DEFAULT(INFO) << "[TensorRT EP] Whole graph will run on TensorRT execution provider";
|
||||
} else {
|
||||
LOGS_DEFAULT(INFO) << "[TensorRT EP] Graph is partitioned and number of subgraphs running on TensorRT exeuction provider is " << number_of_subgraphs;
|
||||
LOGS_DEFAULT(INFO) << "[TensorRT EP] Graph is partitioned and number of subgraphs running on TensorRT execution provider is " << number_of_subgraphs;
|
||||
}
|
||||
|
||||
return result;
|
||||
|
|
@ -1147,18 +1148,18 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
std::vector<NodeComputeInfo>& node_compute_funcs) {
|
||||
for (const auto* fused_node : fused_nodes) {
|
||||
// Build map from input name to its index in input definitions
|
||||
std::unordered_map<std::string, int> input_map;
|
||||
std::unordered_map<std::string, size_t> input_map;
|
||||
const auto& input_defs = fused_node->InputDefs();
|
||||
input_map.reserve(input_defs.size());
|
||||
for (int i = 0, end = input_defs.size(); i < end; ++i) {
|
||||
for (size_t i = 0, end = input_defs.size(); i < end; ++i) {
|
||||
input_map[input_defs[i]->Name()] = i;
|
||||
}
|
||||
|
||||
// Build map from output name to its index in output definitions
|
||||
std::unordered_map<std::string, int> output_map;
|
||||
std::unordered_map<std::string, size_t> output_map;
|
||||
const auto& output_defs = fused_node->OutputDefs();
|
||||
output_map.reserve(output_defs.size());
|
||||
for (int i = 0, end = output_defs.size(); i < end; ++i) {
|
||||
for (size_t i = 0, end = output_defs.size(); i < end; ++i) {
|
||||
output_map[output_defs[i]->Name()] = i;
|
||||
}
|
||||
|
||||
|
|
@ -1193,10 +1194,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
|
||||
int num_inputs = trt_network->getNbInputs();
|
||||
int num_outputs = trt_network->getNbOutputs();
|
||||
std::unordered_map<std::string, int> input_indexes(num_inputs);
|
||||
std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64_t>>> input_shape_ranges;
|
||||
std::unordered_map<std::string, int> output_indexes(num_outputs);
|
||||
std::unordered_map<std::string, int> output_types(num_outputs);
|
||||
std::unordered_map<std::string, size_t> input_indexes(num_inputs);
|
||||
std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> input_shape_ranges;
|
||||
std::unordered_map<std::string, size_t> output_indexes(num_outputs);
|
||||
std::unordered_map<std::string, size_t> output_types(num_outputs);
|
||||
|
||||
// Initialize shape range for dynamic shape tensors
|
||||
bool has_dynamic_shape = false;
|
||||
|
|
@ -1423,17 +1424,17 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
Ort::CustomOpApi ort{*api};
|
||||
TensorrtFuncState* trt_state = reinterpret_cast<TensorrtFuncState*>(state);
|
||||
std::lock_guard<OrtMutex> lock(*(trt_state->tensorrt_mu_ptr));
|
||||
const std::unordered_map<std::string, int>& input_indexes = (trt_state->input_info)[0];
|
||||
const std::unordered_map<std::string, int>& output_indexes = (trt_state->output_info)[0];
|
||||
const std::unordered_map<std::string, int>& output_types = (trt_state->output_info)[1];
|
||||
const std::unordered_map<std::string, size_t>& input_indexes = (trt_state->input_info)[0];
|
||||
const std::unordered_map<std::string, size_t>& output_indexes = (trt_state->output_info)[0];
|
||||
const std::unordered_map<std::string, size_t>& output_types = (trt_state->output_info)[1];
|
||||
auto& shape_ranges = trt_state->input_shape_ranges;
|
||||
auto trt_builder = trt_state->builder->get();
|
||||
auto trt_engine = trt_state->engine->get();
|
||||
auto trt_context = trt_state->context->get();
|
||||
auto trt_profile = &(trt_state->trt_profile);
|
||||
auto alloc = trt_state->scratch_allocator;
|
||||
int num_inputs = input_indexes.size();
|
||||
int num_outputs = output_indexes.size();
|
||||
int num_inputs = static_cast<int>(input_indexes.size());
|
||||
int num_outputs = static_cast<int>(output_indexes.size());
|
||||
bool engine_update = false;
|
||||
std::unordered_set<std::string> input_names;
|
||||
std::unordered_map<std::string, std::vector<int32_t>> tensor_shape_values;
|
||||
|
|
@ -1513,7 +1514,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
// Check and update shape ranges for dynamic shape inputs
|
||||
input_names.insert(input_name);
|
||||
if (shape_ranges.find(input_name) != shape_ranges.end()) {
|
||||
int input_index = 0;
|
||||
size_t input_index = 0;
|
||||
const auto& iter = input_indexes.find(input_name);
|
||||
if (iter != input_indexes.end()) {
|
||||
input_index = iter->second;
|
||||
|
|
@ -1528,7 +1529,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
if (input->isShapeTensor()) {
|
||||
// Get shape values for shape tensor input
|
||||
const auto& tensor_type = ort.GetTensorElementType(tensor_info);
|
||||
int shape_size = nb_dims == 0 ? 1 : tensor_shapes[0];
|
||||
int shape_size = nb_dims == 0 ? 1 : static_cast<int>(tensor_shapes[0]);
|
||||
tensor_shape_values[input_name].resize(shape_size);
|
||||
switch (tensor_type) {
|
||||
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: {
|
||||
|
|
@ -1559,13 +1560,13 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
|
||||
// Update shape ranges
|
||||
std::vector<int32_t> shapes_min(shape_size), shapes_opt(shape_size), shapes_max(shape_size);
|
||||
int shape_range_size = shape_range.size();
|
||||
int shape_range_size = static_cast<int>(shape_range.size());
|
||||
if (shape_size == shape_range_size) {
|
||||
// If shape size matches, check/update shape range
|
||||
for (int j = 0; j < shape_size; ++j) {
|
||||
shapes_min[j] = shape_range[j].first;
|
||||
shapes_opt[j] = shape_range[j].second;
|
||||
shapes_max[j] = shape_range[j].second;
|
||||
shapes_min[j] = static_cast<int32_t>(shape_range[j].first);
|
||||
shapes_opt[j] = static_cast<int32_t>(shape_range[j].second);
|
||||
shapes_max[j] = static_cast<int32_t>(shape_range[j].second);
|
||||
|
||||
const auto& tensor_shape_value = tensor_shape_values[input_name][j];
|
||||
// Update shape range lower bound
|
||||
|
|
@ -1606,21 +1607,21 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
for (int j = 0, end = nb_dims; j < end; ++j) {
|
||||
const auto& tensor_shape = tensor_shapes[j];
|
||||
if (shape_range.find(j) != shape_range.end()) {
|
||||
dims_min.d[j] = shape_range[j].first;
|
||||
dims_opt.d[j] = shape_range[j].second;
|
||||
dims_max.d[j] = shape_range[j].second;
|
||||
dims_min.d[j] = static_cast<int32_t>(shape_range[j].first);
|
||||
dims_opt.d[j] = static_cast<int32_t>(shape_range[j].second);
|
||||
dims_max.d[j] = static_cast<int32_t>(shape_range[j].second);
|
||||
|
||||
// Update minimum dimension
|
||||
if (tensor_shape < shape_range[j].first) {
|
||||
shape_range[j].first = tensor_shape;
|
||||
dims_min.d[j] = tensor_shape;
|
||||
dims_min.d[j] = static_cast<int32_t>(tensor_shape);
|
||||
engine_update = true;
|
||||
}
|
||||
// Update maximum dimension
|
||||
if (tensor_shape > shape_range[j].second) {
|
||||
shape_range[j].second = tensor_shape;
|
||||
dims_max.d[j] = tensor_shape;
|
||||
dims_opt.d[j] = tensor_shape;
|
||||
dims_max.d[j] = static_cast<int32_t>(tensor_shape);
|
||||
dims_opt.d[j] = static_cast<int32_t>(tensor_shape);
|
||||
engine_update = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -1725,14 +1726,14 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
|
||||
// Set input shapes and assign input buffers
|
||||
std::vector<IAllocatorUniquePtr<void>> scratch_buffers;
|
||||
for (int i = 0, end = input_binding_names.size(); i < end; ++i) {
|
||||
for (size_t i = 0, end = input_binding_names.size(); i < end; ++i) {
|
||||
const std::string& input_name = input_binding_names[i];
|
||||
int binding_index = trt_engine->getBindingIndex(input_name.c_str());
|
||||
if (binding_index == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int input_index = 0;
|
||||
size_t input_index = 0;
|
||||
const auto& iter = input_indexes.find(input_name);
|
||||
if (iter != input_indexes.end()) {
|
||||
input_index = iter->second;
|
||||
|
|
@ -1749,7 +1750,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
trt_context->setInputShapeBinding(binding_index, &tensor_shape_values[input_name][0]);
|
||||
} else {
|
||||
for (int j = 0, end = nb_dims; j < end; ++j) {
|
||||
dimensions.d[j] = tensor_shapes[j];
|
||||
dimensions.d[j] = static_cast<int32_t>(tensor_shapes[j]);
|
||||
}
|
||||
trt_context->setBindingDimensions(binding_index, dimensions);
|
||||
}
|
||||
|
|
@ -1862,7 +1863,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
// Set output shapes and assign output buffers
|
||||
std::vector<int> output_dim_sizes(num_outputs, 1);
|
||||
std::vector<OrtValue*> output_tensor(num_outputs, nullptr);
|
||||
for (int i = 0, end = output_binding_names.size(); i < end; ++i) {
|
||||
for (size_t i = 0, end = output_binding_names.size(); i < end; ++i) {
|
||||
// Set dynamic shapes
|
||||
const std::string& output_name = output_binding_names[i];
|
||||
int binding_index = trt_engine->getBindingIndex(output_name.c_str());
|
||||
|
|
@ -1870,7 +1871,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
continue;
|
||||
}
|
||||
|
||||
int output_index = 0;
|
||||
size_t output_index = 0;
|
||||
const auto& index_iter = output_indexes.find(output_name);
|
||||
if (index_iter != output_indexes.end()) {
|
||||
output_index = index_iter->second;
|
||||
|
|
@ -1883,7 +1884,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
}
|
||||
output_tensor[i] = ort.KernelContext_GetOutput(context, output_index, output_shapes.data(), output_shapes.size());
|
||||
|
||||
int output_type = 0;
|
||||
size_t output_type = 0;
|
||||
const auto& type_iter = output_types.find(output_name);
|
||||
if (type_iter != output_types.end()) {
|
||||
output_type = type_iter->second;
|
||||
|
|
@ -1998,10 +1999,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
|
|||
}
|
||||
|
||||
// Cast INT64 input to INT32 because TensorRT doesn't fully support INT64
|
||||
for (int i = 0, end = output_binding_names.size(); i < end; ++i) {
|
||||
for (size_t i = 0, end = output_binding_names.size(); i < end; ++i) {
|
||||
const std::string& output_name = output_binding_names[i];
|
||||
size_t binding_index = trt_engine->getBindingIndex(output_name.c_str());
|
||||
int output_type = 0;
|
||||
size_t output_type = 0;
|
||||
const auto& iter = output_types.find(output_name);
|
||||
if (iter != output_types.end()) {
|
||||
output_type = iter->second;
|
||||
|
|
|
|||
|
|
@ -39,10 +39,16 @@ class TensorrtLogger : public nvinfer1::ILogger {
|
|||
void log(Severity severity, const char* msg) noexcept override {
|
||||
if (severity <= verbosity_) {
|
||||
time_t rawtime = std::time(0);
|
||||
struct tm stm;
|
||||
#ifdef _MSC_VER
|
||||
gmtime_s(&stm, &rawtime);
|
||||
#else
|
||||
gmtime_r(&rawtime, &stm);
|
||||
#endif
|
||||
char buf[256];
|
||||
strftime(&buf[0], 256,
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
std::gmtime(&rawtime));
|
||||
&stm);
|
||||
const char* sevstr = (severity == Severity::kINTERNAL_ERROR ? " BUG" : severity == Severity::kERROR ? " ERROR"
|
||||
: severity == Severity::kWARNING ? "WARNING"
|
||||
: severity == Severity::kINFO ? " INFO"
|
||||
|
|
@ -81,9 +87,9 @@ struct TensorrtFuncState {
|
|||
tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>* context = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>* builder = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::INetworkDefinition>* network = nullptr;
|
||||
std::vector<std::unordered_map<std::string, int>> input_info;
|
||||
std::vector<std::unordered_map<std::string, int>> output_info;
|
||||
std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64_t>>> input_shape_ranges;
|
||||
std::vector<std::unordered_map<std::string, size_t>> input_info;
|
||||
std::vector<std::unordered_map<std::string, size_t>> output_info;
|
||||
std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> input_shape_ranges;
|
||||
OrtMutex* tensorrt_mu_ptr = nullptr;
|
||||
bool fp16_enable;
|
||||
bool int8_enable;
|
||||
|
|
@ -140,7 +146,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
bool external_stream_ = false;
|
||||
cudaStream_t stream_ = nullptr;
|
||||
int max_partition_iterations_ = 1000;
|
||||
int min_subgraph_size_ = 1;
|
||||
size_t min_subgraph_size_ = 1;
|
||||
size_t max_workspace_size_ = 1 << 30; // 1GB
|
||||
bool fp16_enable_ = false;
|
||||
bool int8_enable_ = false;
|
||||
|
|
@ -167,9 +173,9 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>> contexts_;
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>> builders_;
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::INetworkDefinition>> networks_;
|
||||
std::unordered_map<std::string, std::vector<std::unordered_map<std::string, int>>> input_info_;
|
||||
std::unordered_map<std::string, std::vector<std::unordered_map<std::string, int>>> output_info_;
|
||||
std::unordered_map<std::string, std::unordered_map<std::string, std::unordered_map<int, std::pair<int64_t, int64_t>>>> input_shape_ranges_;
|
||||
std::unordered_map<std::string, std::vector<std::unordered_map<std::string, size_t>>> input_info_;
|
||||
std::unordered_map<std::string, std::vector<std::unordered_map<std::string, size_t>>> output_info_;
|
||||
std::unordered_map<std::string, std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>>> input_shape_ranges_;
|
||||
|
||||
/**Get IndexedSubGraph based on node list of the subgraph*/
|
||||
std::unique_ptr<IndexedSubGraph> GetSubGraph(SubGraph_t graph_nodes_index,
|
||||
|
|
|
|||
|
|
@ -88,7 +88,11 @@ struct Tensorrt_Provider : Provider {
|
|||
trt_options.trt_int8_calibration_table_name = nullptr;
|
||||
} else {
|
||||
dest = new char[str_size + 1];
|
||||
#ifdef _MSC_VER
|
||||
strncpy_s(dest, str_size + 1, internal_options.int8_calibration_table_name.c_str(), str_size);
|
||||
#else
|
||||
strncpy(dest, internal_options.int8_calibration_table_name.c_str(), str_size);
|
||||
#endif
|
||||
dest[str_size] = '\0';
|
||||
trt_options.trt_int8_calibration_table_name = (const char*)dest;
|
||||
}
|
||||
|
|
@ -104,7 +108,11 @@ struct Tensorrt_Provider : Provider {
|
|||
trt_options.trt_engine_cache_path = nullptr;
|
||||
} else {
|
||||
dest = new char[str_size + 1];
|
||||
#ifdef _MSC_VER
|
||||
strncpy_s(dest, str_size + 1, internal_options.engine_cache_path.c_str(), str_size);
|
||||
#else
|
||||
strncpy(dest, internal_options.engine_cache_path.c_str(), str_size);
|
||||
#endif
|
||||
dest[str_size] = '\0';
|
||||
trt_options.trt_engine_cache_path = (const char*)dest;
|
||||
}
|
||||
|
|
@ -116,7 +124,11 @@ struct Tensorrt_Provider : Provider {
|
|||
trt_options.trt_engine_decryption_lib_path = nullptr;
|
||||
} else {
|
||||
dest = new char[str_size + 1];
|
||||
#ifdef _MSC_VER
|
||||
strncpy_s(dest, str_size + 1, internal_options.engine_decryption_lib_path.c_str(), str_size);
|
||||
#else
|
||||
strncpy(dest, internal_options.engine_decryption_lib_path.c_str(), str_size);
|
||||
#endif
|
||||
dest[str_size] = '\0';
|
||||
trt_options.trt_engine_decryption_lib_path = (const char*)dest;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue