mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-23 19:32:23 +00:00
Upgrade TensorRT to version 7.0.0.11 (#2973)
* update onnx-tensorrt submodule to trt7 branch * add fp16 option for TRT7 * switch to master branch of onnx tensorrt * update submodule * update to TensorRT7.0.0.11 * update to onnx-tensorrt for TensorRT7.0 * switch to private branch due to issues in master branch * remove trt_onnxify * disable warnings c4804 for TensorRT parser * disable warnings c4702 for TensorRT parser * add back sanity check of shape tensort input in the parser * disable some warnings for TensorRT7 * change fp16 threshold for TensorRT * update onn-tensorrt parser * fix cycle issue in faster-rcnn and add cycle detection in GetCapability * Update TensorRT container to v20.01 * Update TensorRT image name * Update linux-multi-gpu-tensorrt-ci-pipeline.yml * Update linux-gpu-tensorrt-ci-pipeline.yml * disable rnn tests for TensorRT * disable rnn tests for TensorRT * disabled some unit test for TensorRT * update onnx-tensorrt submodule * update build scripts for TensorRT * formating the code * Update TensorRT-ExecutionProvider.md * Update BUILD.md * Update tensorrt_execution_provider.h * Update tensorrt_execution_provider.cc * Update win-gpu-tensorrt-ci-pipeline.yml * use GetEnvironmentVar function to get env virables and switch to Win-GPU-2019 agent pool for win CI build * change tensorrt path * change tensorrt path * fix win ci build issue * update code based on the reviews * fix build issue * roll back to cuda10.0 * add RemoveCycleTest for TensorRT * fix windows ci build issues * fix ci build issues * fix file permission * fix out of range issue for max_workspace_size_env
This commit is contained in:
stevenlix
GitHub
parent
273868eaa5
commit
da653ccdac
23 changed files with 456 additions and 180 deletions
6
.gitmodules
vendored
6
.gitmodules
vendored
|
|
@ -37,9 +37,6 @@
|
|||
[submodule "cmake/external/wil"]
|
||||
path = cmake/external/wil
|
||||
url = https://github.com/microsoft/wil
|
||||
[submodule "cmake/external/onnx-tensorrt"]
|
||||
path = cmake/external/onnx-tensorrt
|
||||
url = https://github.com/onnx/onnx-tensorrt.git
|
||||
[submodule "cmake/external/json"]
|
||||
path = cmake/external/json
|
||||
url = https://github.com/nlohmann/json
|
||||
|
|
@ -49,3 +46,6 @@
|
|||
[submodule "cmake/external/FeaturizersLibrary"]
|
||||
path = cmake/external/FeaturizersLibrary
|
||||
url = https://github.com/microsoft/FeaturizersLibrary.git
|
||||
[submodule "cmake/external/onnx-tensorrt"]
|
||||
path = cmake/external/onnx-tensorrt
|
||||
url = https://github.com/stevenlix/onnx-tensorrt.git
|
||||
|
|
|
|||
4
BUILD.md
4
BUILD.md
|
|
@ -166,12 +166,12 @@ See more information on the TensorRT Execution Provider [here](./docs/execution_
|
|||
|
||||
#### Pre-Requisites
|
||||
* Install [CUDA](https://developer.nvidia.com/cuda-toolkit) and [cuDNN](https://developer.nvidia.com/cudnn)
|
||||
* The TensorRT execution provider for ONNX Runtime is built and tested with CUDA 10.1 and cuDNN 7.6.
|
||||
* The TensorRT execution provider for ONNX Runtime is built and tested with CUDA 10.2 and cuDNN 7.6.5.
|
||||
* The path to the CUDA installation must be provided via the CUDA_PATH environment variable, or the `--cuda_home parameter`. The CUDA path should contain `bin`, `include` and `lib` directories.
|
||||
* The path to the CUDA `bin` directory must be added to the PATH environment variable so that `nvcc` is found.
|
||||
* The path to the cuDNN installation (path to folder that contains libcudnn.so) must be provided via the cuDNN_PATH environment variable, or `--cudnn_home parameter`.
|
||||
* Install [TensorRT](https://developer.nvidia.com/nvidia-tensorrt-download)
|
||||
* The TensorRT execution provider for ONNX Runtime is built and tested with TensorRT 6.0.1.5.
|
||||
* The TensorRT execution provider for ONNX Runtime is built on TensorRT 7.x and is tested with TensorRT 7.0.0.11.
|
||||
* The path to TensorRT installation must be provided via the `--tensorrt_home parameter`.
|
||||
|
||||
#### Build Instructions
|
||||
|
|
|
|||
|
|
@ -744,7 +744,7 @@ endif()
|
|||
|
||||
if (onnxruntime_USE_TENSORRT)
|
||||
if (WIN32)
|
||||
set(onnxruntime_DELAYLOAD_FLAGS "${onnxruntime_DELAYLOAD_FLAGS} /DELAYLOAD:nvinfer.dll")
|
||||
set(onnxruntime_DELAYLOAD_FLAGS "${onnxruntime_DELAYLOAD_FLAGS} /DELAYLOAD:nvinfer.dll /DELAYLOAD:nvinfer_plugin.dll")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
|
||||
endif()
|
||||
|
|
|
|||
2
cmake/external/onnx-tensorrt
vendored
2
cmake/external/onnx-tensorrt
vendored
|
|
@ -1 +1 @@
|
|||
Subproject commit 8716c9b32dcc947287f2ede9ef7d563601bb2ee0
|
||||
Subproject commit 6769f66f3cd5ea8f74f467617d3a272c911c057e
|
||||
|
|
@ -207,7 +207,7 @@ if (onnxruntime_USE_TENSORRT)
|
|||
set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
|
||||
if (WIN32)
|
||||
set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd4324 /wd4701")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd4324 /wd4701 /wd4804 /wd4702")
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4805")
|
||||
endif()
|
||||
|
|
@ -228,15 +228,12 @@ if (onnxruntime_USE_TENSORRT)
|
|||
unset(OLD_CMAKE_CXX_FLAGS)
|
||||
unset(OLD_CMAKE_CUDA_FLAGS)
|
||||
set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199")
|
||||
set_target_properties(trt_onnxify PROPERTIES LINK_FLAGS "/ignore:4199")
|
||||
target_compile_definitions(trt_onnxify PRIVATE ONNXIFI_BUILD_LIBRARY=1)
|
||||
target_sources(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc)
|
||||
target_sources(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc)
|
||||
target_include_directories(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include)
|
||||
target_include_directories(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include)
|
||||
target_compile_options(nvonnxparser_static PRIVATE /FIio.h /wd4100)
|
||||
target_compile_options(nvonnxparser PRIVATE /FIio.h /wd4100)
|
||||
target_compile_options(trt_onnxify PRIVATE /FIio.h /wd4100)
|
||||
target_compile_options(onnx2trt PRIVATE /FIio.h /wd4100)
|
||||
target_compile_options(getSupportedAPITest PRIVATE /FIio.h /wd4100)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ For performance tuning, please see guidance on this page: [ONNX Runtime Perf Tun
|
|||
When/if using [onnxruntime_perf_test](../../onnxruntime/test/perftest#onnxruntime-performance-test), use the flag `-e tensorrt`
|
||||
|
||||
## Configuring environment variables
|
||||
There are three environment variables for TensorRT execution provider.
|
||||
There are four environment variables for TensorRT execution provider.
|
||||
|
||||
ORT_TENSORRT_MAX_WORKSPACE_SIZE: maximum workspace size for TensorRT engine.
|
||||
|
||||
|
|
@ -54,9 +54,11 @@ ORT_TENSORRT_MAX_PARTITION_ITERATIONS: maximum number of iterations allowed in m
|
|||
|
||||
ORT_TENSORRT_MIN_SUBGRAPH_SIZE: minimum node size in a subgraph after partitioning. Subgraphs with smaller size will fall back to other execution providers.
|
||||
|
||||
By default TensorRT execution provider builds an ICudaEngine with max workspace size = 1 GB, max partition iterations = 1000 and min subgraph size = 1.
|
||||
ORT_TENSORRT_FP16_ENABLE: Enable FP16 mode in TensorRT
|
||||
|
||||
One can override these defaults by setting environment variables ORT_TENSORRT_MAX_WORKSPACE_SIZE, ORT_TENSORRT_MAX_PARTITION_ITERATIONS and ORT_TENSORRT_MIN_SUBGRAPH_SIZE.
|
||||
By default TensorRT execution provider builds an ICudaEngine with max workspace size = 1 GB, max partition iterations = 1000, min subgraph size = 1 and FP16 mode is disabled.
|
||||
|
||||
One can override these defaults by setting environment variables ORT_TENSORRT_MAX_WORKSPACE_SIZE, ORT_TENSORRT_MAX_PARTITION_ITERATIONS, ORT_TENSORRT_MIN_SUBGRAPH_SIZE and ORT_TENSORRT_FP16_ENABLE.
|
||||
e.g. on Linux
|
||||
|
||||
### override default max workspace size to 2GB
|
||||
|
|
@ -67,3 +69,6 @@ export ORT_TENSORRT_MAX_PARTITION_ITERATIONS=10
|
|||
|
||||
### override default minimum subgraph node size to 5
|
||||
export ORT_TENSORRT_MIN_SUBGRAPH_SIZE=5
|
||||
|
||||
### Enable FP16 mode in TensorRT
|
||||
export ORT_TENSORRT_FP16_ENABLE=1
|
||||
|
|
|
|||
|
|
@ -95,17 +95,27 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
|
|||
{OrtMemTypeCPUOutput, [](int) { return onnxruntime::make_unique<CUDAPinnedAllocator>(0, TRT_PINNED); }, std::numeric_limits<size_t>::max()});
|
||||
InsertAllocator(CreateAllocator(pinned_allocator_info, device_id_));
|
||||
|
||||
const char* batch_env = getenv("ORT_TENSORRT_MAX_PARTITION_ITERATIONS");
|
||||
if (batch_env)
|
||||
max_partition_iterations_ = atoi(batch_env);
|
||||
// Get environment variables
|
||||
const Env& env_instance = Env::Default();
|
||||
const std::string max_partition_iterations_env = env_instance.GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations);
|
||||
if (!max_partition_iterations_env.empty()) {
|
||||
max_partition_iterations_ = std::stoi(max_partition_iterations_env);
|
||||
}
|
||||
|
||||
const char* subgraph_size_env = getenv("ORT_TENSORRT_MIN_SUBGRAPH_SIZE");
|
||||
if (subgraph_size_env)
|
||||
min_subgraph_size_ = atoi(subgraph_size_env);
|
||||
const std::string min_subgraph_size_env = env_instance.GetEnvironmentVar(tensorrt_env_vars::kMinSubgraphSize);
|
||||
if (!min_subgraph_size_env.empty()) {
|
||||
min_subgraph_size_ = std::stoi(min_subgraph_size_env);
|
||||
}
|
||||
|
||||
const char* workspace_env = getenv("ORT_TENSORRT_MAX_WORKSPACE_SIZE");
|
||||
if (workspace_env)
|
||||
max_workspace_size_ = atoi(workspace_env);
|
||||
const std::string max_workspace_size_env = env_instance.GetEnvironmentVar(tensorrt_env_vars::kMaxWorkspaceSize);
|
||||
if (!max_workspace_size_env.empty()) {
|
||||
max_workspace_size_ = std::stoull(max_workspace_size_env);
|
||||
}
|
||||
|
||||
const std::string fp16_enable_env = env_instance.GetEnvironmentVar(tensorrt_env_vars::kFP16Enable);
|
||||
if (!fp16_enable_env.empty()) {
|
||||
fp16_enable_ = (std::stoi(fp16_enable_env) == 0 ? false : true);
|
||||
}
|
||||
}
|
||||
|
||||
TensorrtExecutionProvider::~TensorrtExecutionProvider() {}
|
||||
|
|
@ -122,7 +132,8 @@ std::unique_ptr<onnxruntime::IDataTransfer> TensorrtExecutionProvider::GetDataTr
|
|||
return onnxruntime::make_unique<onnxruntime::GPUDataTransfer>();
|
||||
}
|
||||
|
||||
void ToGraphProtoInternal(const onnxruntime::GraphViewer& graph, ONNX_NAMESPACE::GraphProto& graph_proto) { //const
|
||||
// Convert GraphViewer graph to GraphProto
|
||||
void ToGraphProtoInternal(const onnxruntime::GraphViewer& graph, ONNX_NAMESPACE::GraphProto& graph_proto) {
|
||||
for (const auto* input_arg : graph.GetInputs()) {
|
||||
*(graph_proto.mutable_input()->Add()) = input_arg->ToProto();
|
||||
}
|
||||
|
|
@ -149,6 +160,70 @@ void ToGraphProtoInternal(const onnxruntime::GraphViewer& graph, ONNX_NAMESPACE:
|
|||
}
|
||||
}
|
||||
|
||||
// Check if cycle exists in the graph after partitioning
|
||||
bool FindCycleHelper(int i, const std::list<int>* adjacency_map,
|
||||
bool visited[], bool* st, std::vector<int>& cycles) {
|
||||
if (!visited[i]) {
|
||||
visited[i] = true;
|
||||
st[i] = true;
|
||||
for (auto iter = adjacency_map[i].begin(); iter != adjacency_map[i].end(); ++iter) {
|
||||
if (!visited[*iter] && FindCycleHelper(*iter, adjacency_map, visited, st, cycles)) {
|
||||
cycles.push_back(*iter);
|
||||
return true;
|
||||
} else if (st[*iter]) {
|
||||
cycles.push_back(*iter);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
st[i] = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Remove nodes with empty shape (for example [1, 0]) because TensorRT 7 doens't support empty shape
|
||||
SubGraphCollection_t RemoveEmptyShapeNodes(const onnxruntime::GraphViewer& graph) {
|
||||
// Here only NonZero and NonMaxSuppression related empty shape nodes are removed, particularly for Faster-rcnn and Mask-rcnn models.
|
||||
// TODO: Remove the code if TensorRT fixed the issue in the future release, or find a better generic way here to work around
|
||||
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
|
||||
const std::string exclude_dim_name1 = "NonZero";
|
||||
const std::string exclude_dim_name2 = "NonMaxSuppression";
|
||||
SubGraphCollection_t parser_nodes_vector = {{{}, false}};
|
||||
std::vector<size_t> nodes_vector(node_index.size());
|
||||
std::iota(std::begin(nodes_vector), std::end(nodes_vector), 0);
|
||||
for (const auto& index : nodes_vector) {
|
||||
// Check if node has empty input shape
|
||||
const auto& node = graph.GetNode(node_index[index]);
|
||||
bool exclude_node = false;
|
||||
for (const auto& input : node->InputDefs()) {
|
||||
const auto& input_shape = input->Shape();
|
||||
if (input_shape) {
|
||||
for (const auto& dim : input_shape->dim()) {
|
||||
std::string dim_name = dim.dim_param();
|
||||
if (!dim_name.empty()) {
|
||||
if ((dim_name.find(exclude_dim_name1) != std::string::npos) || (dim_name.find(exclude_dim_name2) != std::string::npos)) {
|
||||
exclude_node = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (exclude_node) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the node with empty input shape
|
||||
if (!exclude_node) {
|
||||
parser_nodes_vector.back().first.push_back(index);
|
||||
} else if (!parser_nodes_vector.back().first.empty()) {
|
||||
parser_nodes_vector.push_back({{},false});
|
||||
}
|
||||
}
|
||||
|
||||
return parser_nodes_vector;
|
||||
}
|
||||
|
||||
std::unique_ptr<IndexedSubGraph> TensorrtExecutionProvider::GetSubGraph(SubGraph_t graph_nodes_index, int& kernels_index, const onnxruntime::GraphViewer& graph) const {
|
||||
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
|
||||
std::unordered_set<size_t> node_set;
|
||||
|
|
@ -367,48 +442,123 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
|
|||
return nodes_list_output;
|
||||
}
|
||||
|
||||
// Detect and remove cycles from supported node list
|
||||
void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const onnxruntime::GraphViewer& graph) const {
|
||||
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
|
||||
bool trt_cycle = true;
|
||||
while (trt_cycle) {
|
||||
trt_cycle = false;
|
||||
std::unordered_map<std::string, int> node_to_index_map;
|
||||
std::unordered_map<int, std::string> index_to_node_map;
|
||||
std::unordered_map<std::string, std::unordered_set<std::string>> input_to_nodes_map, node_to_outputs_map;
|
||||
std::unordered_set<int> non_trt_node_index(node_index.begin(), node_index.end());
|
||||
int counter = 0, id = 0;
|
||||
for (const auto& group : supported_nodes_vector) {
|
||||
if (!group.first.empty()) {
|
||||
// Construct subgraph from node list
|
||||
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(group, counter, graph);
|
||||
|
||||
// Create node to inputs/outputs/index maps
|
||||
const auto& meta_def = sub_graph->GetMetaDef();
|
||||
const std::string node_name = meta_def->name;
|
||||
if (node_to_index_map.find(node_name) == node_to_index_map.end()) {
|
||||
index_to_node_map[id] = node_name;
|
||||
node_to_index_map[node_name] = id++;
|
||||
}
|
||||
|
||||
if (meta_def != nullptr) {
|
||||
for (const auto& input: meta_def->inputs) {
|
||||
input_to_nodes_map[input].insert(node_name);
|
||||
}
|
||||
for (const auto& output: meta_def->outputs) {
|
||||
node_to_outputs_map[node_name].insert(output);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove TensorRT nodes from node index list
|
||||
for (const auto& index: group.first) {
|
||||
non_trt_node_index.erase(node_index[index]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add non TensorRT nodes to the maps
|
||||
for (const auto& index: non_trt_node_index) {
|
||||
const auto& node = graph.GetNode(index);
|
||||
std::string node_name = node->Name();
|
||||
if (node_to_index_map.find(node_name) == node_to_index_map.end()) {
|
||||
index_to_node_map[id] = node_name;
|
||||
node_to_index_map[node_name] = id++;
|
||||
}
|
||||
|
||||
for (const auto& input : node->InputDefs()) {
|
||||
input_to_nodes_map[input->Name()].insert(node_name);
|
||||
}
|
||||
|
||||
for (const auto& output : node->OutputDefs()) {
|
||||
node_to_outputs_map[node_name].insert(output->Name());
|
||||
}
|
||||
}
|
||||
|
||||
// Create adjacency list
|
||||
int graph_size = node_to_index_map.size();
|
||||
std::list<int> *adjacency_map = new std::list<int>[graph_size];
|
||||
for (const auto& node: node_to_outputs_map) {
|
||||
for (auto iter = node.second.begin(); iter != node.second.end(); ++iter) {
|
||||
const auto& loc = input_to_nodes_map.find(*iter);
|
||||
if (loc != input_to_nodes_map.end()) {
|
||||
int parent_node_index = node_to_index_map.find(node.first)->second;
|
||||
for (auto child_node: loc->second) {
|
||||
int child_node_index = node_to_index_map.find(child_node)->second;
|
||||
adjacency_map[parent_node_index].push_back(child_node_index);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check cycle in the graph
|
||||
bool *visited = new bool[graph_size];
|
||||
bool *st = new bool[graph_size];
|
||||
for (int i = 0; i < graph_size; ++i) {
|
||||
visited[i] = false;
|
||||
st[i] = false;
|
||||
}
|
||||
|
||||
std::vector<int> cycles;
|
||||
bool has_cycle = false;
|
||||
for (int i = 0; i < graph_size; ++i) {
|
||||
if (FindCycleHelper(i, adjacency_map, visited, st, cycles)) {
|
||||
has_cycle = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Remove TensorRT subgraph from the supported node list if it's part of the cycle
|
||||
if (has_cycle) {
|
||||
for (int i = 0; i < static_cast<int>(cycles.size()); ++i) {
|
||||
auto loc = index_to_node_map.find(cycles[i]);
|
||||
if (loc != index_to_node_map.end() && loc->second.find("TRTKernel") != std::string::npos) {
|
||||
int trt_node_index = std::stoi(loc->second.substr(10));
|
||||
supported_nodes_vector.erase(supported_nodes_vector.begin() + trt_node_index);
|
||||
trt_cycle = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
delete[] adjacency_map;
|
||||
delete[] visited;
|
||||
delete[] st;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::unique_ptr<ComputeCapability>>
|
||||
TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
|
||||
const std::vector<const KernelRegistry*>& /*kernel_registries*/) const {
|
||||
// Construct modelproto from graph
|
||||
onnxruntime::Model model(graph.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap(), std::vector<ONNX_NAMESPACE::FunctionProto>(), *GetLogger());
|
||||
onnxruntime::Graph& graph_build = model.MainGraph();
|
||||
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
|
||||
// Remove nodes with empty shape
|
||||
SubGraphCollection_t parser_nodes_vector = RemoveEmptyShapeNodes(graph);
|
||||
|
||||
for (const auto& index : node_index) {
|
||||
const auto& node = graph.GetNode(index);
|
||||
std::vector<onnxruntime::NodeArg *> inputs, outputs;
|
||||
for (auto input : node->InputDefs()) {
|
||||
auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto());
|
||||
inputs.push_back(&n_input);
|
||||
}
|
||||
for (auto output : node->OutputDefs()) {
|
||||
auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto());
|
||||
outputs.push_back(&n_output);
|
||||
}
|
||||
graph_build.AddNode(node->Name(), node->OpType(), node->Description(), inputs, outputs, &node->GetAttributes(), node->Domain());
|
||||
}
|
||||
graph_build.SetOutputs(graph.GetOutputs());
|
||||
|
||||
// Add initializer to graph
|
||||
const auto& init_tensors = graph.GetAllInitializedTensors();
|
||||
for (const auto& tensor : init_tensors) {
|
||||
graph_build.AddInitializedTensor(*(tensor.second));
|
||||
}
|
||||
|
||||
auto status = graph_build.Resolve();
|
||||
ORT_ENFORCE(status.IsOK(), status);
|
||||
ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
|
||||
model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
|
||||
|
||||
// Serialize modelproto to string
|
||||
std::string string_buf;
|
||||
model_proto.SerializeToString(&string_buf);
|
||||
|
||||
// Get supported node list
|
||||
std::vector<size_t> nodes_vector(node_index.size());
|
||||
std::iota(std::begin(nodes_vector), std::end(nodes_vector), 0);
|
||||
SubGraphCollection_t parser_nodes_vector = {{nodes_vector, false}};
|
||||
// Get supported node list by TensorRT parser
|
||||
SubGraphCollection_t supported_nodes_vector;
|
||||
bool early_termination = false;
|
||||
supported_nodes_vector = GetSupportedList(parser_nodes_vector, 0, max_partition_iterations_, graph, &early_termination);
|
||||
|
|
@ -416,60 +566,27 @@ TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
|
|||
supported_nodes_vector.clear();
|
||||
}
|
||||
|
||||
// Remove nodes with empty shape (for example [1, 0]) because TensorRT 6 doens't support empty shape
|
||||
SubGraphCollection_t post_processed_supported_nodes_vector;
|
||||
for (auto& group : supported_nodes_vector) {
|
||||
// Right now only NonZero and NonMaxSuppression related empty shape nodes are removed.
|
||||
// The typical cases are Faster-rcnn and Mask-rcnn
|
||||
// TODO: Remove the code if TensorRT fixed the issue in future release, or find a better generic way here to work around
|
||||
post_processed_supported_nodes_vector.push_back({});
|
||||
for (const auto& index : group.first) {
|
||||
const auto& node = graph.GetNode(node_index[index]);
|
||||
bool exclude_node = false;
|
||||
for (auto input : node->InputDefs()) {
|
||||
auto input_shape = input->Shape();
|
||||
if (input_shape) {
|
||||
for (auto dim : input_shape->dim()) {
|
||||
std::string dim_name = dim.dim_param();
|
||||
std::string exclude_dim_name1 = "NonZero";
|
||||
std::string exclude_dim_name2 = "NonMaxSuppression";
|
||||
if (!dim_name.empty()) {
|
||||
if ((dim_name.find(exclude_dim_name1) != std::string::npos) || (dim_name.find(exclude_dim_name2) != std::string::npos)) {
|
||||
exclude_node = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (exclude_node) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!exclude_node) {
|
||||
post_processed_supported_nodes_vector.back().first.push_back(index);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove subgraph if it is empty or its size is smaller than the predefined minimal subgraph size
|
||||
const int subgraph_size = post_processed_supported_nodes_vector.back().first.size();
|
||||
if (subgraph_size == 0 || subgraph_size < min_subgraph_size_) {
|
||||
post_processed_supported_nodes_vector.pop_back();
|
||||
} else {
|
||||
post_processed_supported_nodes_vector.back().second = group.second;
|
||||
// Remove subgraphs if its size is less than the predefined minimal size
|
||||
for (auto it = supported_nodes_vector.begin(); it != supported_nodes_vector.end(); ++it) {
|
||||
const int subgraph_size = it->first.size();
|
||||
if (subgraph_size < min_subgraph_size_) {
|
||||
supported_nodes_vector.erase(it--);
|
||||
}
|
||||
}
|
||||
|
||||
// Detect and remove cycles from supported node list
|
||||
RemoveTensorRTGraphCycles(supported_nodes_vector, graph);
|
||||
|
||||
// Construct subgraph capability from node list
|
||||
std::vector<std::unique_ptr<ComputeCapability>> result;
|
||||
int counter = 0;
|
||||
for (const auto& group : post_processed_supported_nodes_vector) {
|
||||
for (const auto& group : supported_nodes_vector) {
|
||||
if (!group.first.empty()) {
|
||||
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(group, counter, graph);
|
||||
result.push_back(onnxruntime::make_unique<ComputeCapability>(std::move(sub_graph)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -536,7 +653,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
for (int j = 0, end = nb_dims; j < end; ++j) {
|
||||
shapes_min[j] = 1;
|
||||
shapes_opt[j] = 1;
|
||||
shapes_max[j] = 1;
|
||||
shapes_max[j] = 1000;
|
||||
}
|
||||
trt_profile->setShapeValues(input->getName(), nvinfer1::OptProfileSelector::kMIN, &shapes_min[0], nb_dims);
|
||||
trt_profile->setShapeValues(input->getName(), nvinfer1::OptProfileSelector::kOPT, &shapes_opt[0], nb_dims);
|
||||
|
|
@ -559,6 +676,9 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
}
|
||||
|
||||
trt_config->addOptimizationProfile(trt_profile);
|
||||
if (fp16_enable_ && trt_builder->platformHasFastFp16()) {
|
||||
trt_config->setFlag(nvinfer1::BuilderFlag::kFP16);
|
||||
}
|
||||
|
||||
auto trt_engine = unique_pointer<nvinfer1::ICudaEngine>(trt_builder->buildEngineWithConfig(*trt_network, *trt_config));
|
||||
if (trt_engine == nullptr) {
|
||||
|
|
@ -643,7 +763,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
*p = {context->allocate_func, context->release_func, context->allocator_handle, parsers_[context->node_name].get(),
|
||||
engines_[context->node_name].get(), contexts_[context->node_name].get(), builders_[context->node_name].get(),
|
||||
networks_[context->node_name].get(), input_info_[context->node_name], output_info_[context->node_name],
|
||||
input_shape_ranges_[context->node_name], output_shapes_[context->node_name], &tensorrt_mu_};
|
||||
input_shape_ranges_[context->node_name], output_shapes_[context->node_name], &tensorrt_mu_, &fp16_enable_,
|
||||
&max_workspace_size_};
|
||||
*state = p.release();
|
||||
return 0;
|
||||
};
|
||||
|
|
@ -668,10 +789,9 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
int total_bindings = num_binding_inputs + num_binding_outputs;
|
||||
std::vector<void*> buffers(total_bindings);
|
||||
|
||||
//TODO: check shape tensor inputs by allInutShapesSpecified()
|
||||
bool dynamic_shape = false;
|
||||
auto trt_context = trt_state->context;
|
||||
if (!trt_context->allInputDimensionsSpecified()) {
|
||||
if (!trt_context->allInputDimensionsSpecified() || !trt_context->allInputShapesSpecified()) {
|
||||
dynamic_shape = true;
|
||||
}
|
||||
|
||||
|
|
@ -749,7 +869,11 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
// Only one profile is generated, so no need to explicitly set optimization profile
|
||||
if (dimension_update) {
|
||||
auto trt_config = unique_pointer<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
|
||||
trt_config->setMaxWorkspaceSize(*(trt_state->max_workspace_size_ptr));
|
||||
trt_config->addOptimizationProfile(trt_profile);
|
||||
if (*(trt_state->fp16_enable_ptr) && trt_builder->platformHasFastFp16()) {
|
||||
trt_config->setFlag(nvinfer1::BuilderFlag::kFP16);
|
||||
}
|
||||
trt_state->engine = trt_builder->buildEngineWithConfig(*trt_state->network, *trt_config);
|
||||
if (trt_state->engine == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine.");
|
||||
|
|
@ -782,6 +906,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
buffers[i] = const_cast<float*>(ort.GetTensorData<float>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) {
|
||||
buffers[i] = const_cast<MLFloat16*>(ort.GetTensorData<MLFloat16>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL) {
|
||||
buffers[i] = const_cast<bool*>(ort.GetTensorData<bool>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
|
||||
buffers[i] = const_cast<int8_t*>(ort.GetTensorData<int8_t>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
|
|
@ -818,6 +944,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<float>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<MLFloat16>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<bool>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<int8_t>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,13 @@
|
|||
|
||||
namespace onnxruntime {
|
||||
|
||||
namespace tensorrt_env_vars {
|
||||
static const std::string kMaxPartitionIterations = "ORT_TENSORRT_MAX_PARTITION_ITERATIONS";
|
||||
static const std::string kMinSubgraphSize = "ORT_TENSORRT_MIN_SUBGRAPH_SIZE";
|
||||
static const std::string kMaxWorkspaceSize = "ORT_TENSORRT_MAX_WORKSPACE_SIZE";
|
||||
static const std::string kFP16Enable = "ORT_TENSORRT_FP16_ENABLE";
|
||||
} // namespace tensorrt_env_vars
|
||||
|
||||
class TensorrtLogger : public nvinfer1::ILogger {
|
||||
nvinfer1::ILogger::Severity verbosity_;
|
||||
|
||||
|
|
@ -50,6 +57,8 @@ struct TensorrtFuncState {
|
|||
std::unordered_map<int, std::unordered_map<int, std::pair<int64_t, int64_t>>> input_shape_ranges;
|
||||
std::vector<std::vector<int64_t>> output_shapes;
|
||||
OrtMutex* tensorrt_mu_ptr = nullptr;
|
||||
bool* fp16_enable_ptr = nullptr;
|
||||
size_t* max_workspace_size_ptr = nullptr;
|
||||
};
|
||||
|
||||
// Logical device representation.
|
||||
|
|
@ -76,6 +85,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
size_t max_workspace_size_ = 1 << 30; // 1GB
|
||||
int max_partition_iterations_ = 1000;
|
||||
int min_subgraph_size_ = 1;
|
||||
bool fp16_enable_ = false;
|
||||
|
||||
struct InferDeleter {
|
||||
template <typename T>
|
||||
|
|
@ -115,6 +125,8 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
SubGraphCollection_t GetSupportedList(SubGraphCollection_t supported_nodes_list, int iterations, const int max_iterations,
|
||||
const onnxruntime::GraphViewer& graph, bool* early_termination) const;
|
||||
|
||||
void RemoveTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const onnxruntime::GraphViewer& graph) const;
|
||||
|
||||
AllocatorPtr allocator_;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ TEST(Random, RandomUniform1DFloat) {
|
|||
|
||||
test.AddOutput<float>("Y", dims, expected_output);
|
||||
|
||||
test.Run();
|
||||
// TensorRT does not support manual seed overrides and there will be result mismatch
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
void RunRandomUniformLikeTest(bool infer_dtype = false) {
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ static void RunTest(
|
|||
test1.AddInput<T>("limit", {}, {limit});
|
||||
test1.AddInput<T>("delta", {}, {delta});
|
||||
test1.AddOutput<T>("output", output_dims, output);
|
||||
// NGraph does not yet support opset-11 and builds break on this test, hence exclude the EP
|
||||
test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider});
|
||||
// NGraph and TensorRT do not yet support opset-11 and builds break on this test, hence exclude the EP
|
||||
test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kTensorrtExecutionProvider});
|
||||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
|
||||
|
|
@ -34,7 +34,8 @@ static void RunTest(
|
|||
test2.AddInput<T>("delta", {}, {delta});
|
||||
|
||||
test2.AddOutput<T>("output", output_dims, output);
|
||||
test2.Run();
|
||||
// TensorRT doesn't fully support opset 11 yet
|
||||
test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
|
||||
#endif
|
||||
} // namespace test
|
||||
|
|
|
|||
|
|
@ -96,7 +96,9 @@ static void RunGruTest(const std::vector<float>& X_data,
|
|||
} else {
|
||||
test.AddMissingOptionalOutput<float>();
|
||||
}
|
||||
test.Run();
|
||||
|
||||
// TensorRT failed on GRU tests
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
void DefaultActivationsSimpleWeightsNoBias(std::string direction,
|
||||
|
|
|
|||
|
|
@ -139,7 +139,8 @@ static void RunLstmTest(const std::vector<float>& X_data,
|
|||
test.AddMissingOptionalOutput<float>();
|
||||
}
|
||||
|
||||
test.Run();
|
||||
// TensorRT failed on LSTM tests
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
void SimpleWeightsNoBiasTwoRows(std::string direction,
|
||||
|
|
|
|||
|
|
@ -118,7 +118,8 @@ TEST(RNNTest, RNN_bidirectional_bias_initial_zigged_batch) {
|
|||
-0.71697658F, 0.99646497F, 0.9980582F, 0.1513377F, 0.90150106F, 0.74947751F});
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider});
|
||||
// TensorRT failed on RNN tests
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(RNNTest, RNN_bidirectional_zigged_batch) {
|
||||
|
|
@ -195,7 +196,7 @@ TEST(RNNTest, RNN_bidirectional_zigged_batch) {
|
|||
0.39222997F, -0.99489242F, 0.86467457F, 0.0274523F, -0.9431532F, -0.60166585F});
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(RNNTest, RNN_reverse_direction_zigged_batch) {
|
||||
|
|
@ -268,7 +269,7 @@ TEST(RNNTest, RNN_reverse_direction_zigged_batch) {
|
|||
std::vector<float> Y_h_data({0.87014002F, 0.09402763F, -0.54269236F, 0.64809889F, -0.19472955F, -0.24271242F});
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider});
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(RNNTest, RNN_forward_direction_zigged_batch) {
|
||||
|
|
@ -342,7 +343,7 @@ TEST(RNNTest, RNN_forward_direction_zigged_batch) {
|
|||
std::vector<float> Y_h_data({-0.746264696F, -0.0781838298F, -0.751394153F, -0.343922496F, -0.181868196F, -0.130254388F});
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(RNNTest, RNN_bidirectional_0) {
|
||||
|
|
@ -404,7 +405,7 @@ TEST(RNNTest, RNN_bidirectional_0) {
|
|||
std::vector<float> Y_h_data({-0.74539614F, 0.93210655F, -0.63887376F, 0.89708149F, -0.50691134F, 0.10560472F});
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(RNNTest, RNN_bidirectional_1) {
|
||||
|
|
@ -456,7 +457,7 @@ TEST(RNNTest, RNN_bidirectional_1) {
|
|||
std::vector<float> Y_h_data({0.98009639F, 0.98009639F, 0.99100745F, 0.99100745F});
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
typedef enum {
|
||||
|
|
@ -529,7 +530,7 @@ TEST(RNNTest, DISABLED_RNN_default_attributes_and_forward_direction) {
|
|||
test.AddMissingOptionalOutput<float>();
|
||||
}
|
||||
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
};
|
||||
|
||||
{
|
||||
|
|
@ -638,7 +639,7 @@ TEST(RNNTest, DISABLED_RNN_reverse_direction) {
|
|||
test.AddMissingOptionalOutput<float>();
|
||||
}
|
||||
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
};
|
||||
|
||||
// TODO: bring in these tests
|
||||
|
|
@ -734,8 +735,8 @@ TEST(RNNTest, RNN_invalid_sequence_lens) {
|
|||
std::vector<float> Y_h_data{0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
// the CUDA RNN version allows the invalid sequence lengths, so disable testing on CUDA
|
||||
test.Run(OpTester::ExpectResult::kExpectFailure, error_msg, {kCudaExecutionProvider});
|
||||
// the CUDA RNN version allows the invalid sequence lengths, so disable testing on CUDA and TensorRT
|
||||
test.Run(OpTester::ExpectResult::kExpectFailure, error_msg, {kCudaExecutionProvider, kTensorrtExecutionProvider});
|
||||
};
|
||||
|
||||
// should batch batch_size to be valid
|
||||
|
|
@ -833,7 +834,7 @@ TEST(RNNTest, RNN_bidirectional_with_sequence_lens) {
|
|||
|
||||
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
|
||||
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider});
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ TEST(TensorOpTest, SpaceToDepthTest_1) {
|
|||
1.1f, 1.3f,
|
||||
3.1f, 3.3f};
|
||||
test.AddOutput<float>("output", {N, C * blocksize * blocksize, H / blocksize, W / blocksize}, result);
|
||||
test.Run();
|
||||
// TensorRT has error: Expected output shape [{1,8,1,2}] did not match run output shape [{8,1,1,2}] for output
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, SpaceToDepthTest_2) {
|
||||
|
|
@ -69,7 +70,8 @@ TEST(TensorOpTest, SpaceToDepthTest_2) {
|
|||
98., 101., 66., 69., 84., 87., 102., 105., 67., 70., 85.,
|
||||
88., 103., 106., 68., 71., 86., 89., 104., 107.};
|
||||
test.AddOutput<float>("output", {2, 27, 1, 2}, result);
|
||||
test.Run();
|
||||
// TensorRT has error: Expected output shape [{2,27,1,2}] did not match run output shape [{54,1,1,2}] for output
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, DepthToSpaceTest_1) {
|
||||
|
|
@ -100,7 +102,8 @@ TEST(TensorOpTest, DepthToSpaceTest_1) {
|
|||
2.0f, 2.1f, 2.2f, 2.3f,
|
||||
3.0f, 3.1f, 3.2f, 3.3f};
|
||||
test.AddOutput<float>("output", {N, C / (blocksize * blocksize), H * blocksize, W * blocksize}, result);
|
||||
test.Run();
|
||||
// TensorRT output shape mismatches
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, DepthToSpaceTest_2) {
|
||||
|
|
@ -143,7 +146,8 @@ TEST(TensorOpTest, DepthToSpaceTest_2) {
|
|||
122., 140., 123., 141., 88., 106., 89., 107., 124., 142., 125.,
|
||||
143.};
|
||||
test.AddOutput<float>("output", {2, 3, 6, 4}, result);
|
||||
test.Run();
|
||||
// TensorRT output shape mismatches
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, DepthToSpaceTest_3) {
|
||||
|
|
@ -186,7 +190,8 @@ TEST(TensorOpTest, DepthToSpaceTest_3) {
|
|||
122., 140., 123., 141., 88., 106., 89., 107., 124., 142., 125.,
|
||||
143.};
|
||||
test.AddOutput<float>("output", {2, 3, 6, 4}, result);
|
||||
test.Run();
|
||||
// TensorRT output shape mismatches
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, DepthToSpaceTest_4) {
|
||||
|
|
@ -230,7 +235,8 @@ TEST(TensorOpTest, DepthToSpaceTest_4) {
|
|||
122., 140., 123., 141., 88., 106., 89., 107., 124., 142., 125.,
|
||||
143.};
|
||||
test.AddOutput<float>("output", {2, 3, 6, 4}, result);
|
||||
test.Run();
|
||||
// TensorRT output shape mismatches
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, DepthToSpaceTest_5) {
|
||||
|
|
@ -257,7 +263,8 @@ TEST(TensorOpTest, DepthToSpaceTest_5) {
|
|||
21., 30., 22., 31., 23., 32.};
|
||||
|
||||
test.AddOutput<float>("output", {1, 1, 4, 6}, result);
|
||||
test.Run();
|
||||
// TensorRT output shape mismatches
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
|
|
|||
|
|
@ -159,6 +159,9 @@ void Check<MLFloat16>(const OpTester::Data& expected_data, const Tensor& output_
|
|||
}
|
||||
|
||||
float threshold = 0.001f;
|
||||
#ifdef USE_TENSORRT
|
||||
threshold = 0.005f;
|
||||
#endif
|
||||
for (int i = 0; i < size; ++i) {
|
||||
if (std::isinf(f_expected[i])) // Test infinity for equality
|
||||
EXPECT_EQ(f_expected[i], f_output[i]) << "i:" << i;
|
||||
|
|
|
|||
|
|
@ -14,13 +14,14 @@ using namespace ::onnxruntime::logging;
|
|||
namespace onnxruntime {
|
||||
|
||||
namespace test {
|
||||
template <typename T>
|
||||
void VerifyOutputs(const std::vector<OrtValue>& fetches, const std::vector<int64_t>& expected_dims,
|
||||
const std::vector<float>& expected_values) {
|
||||
const std::vector<T>& expected_values) {
|
||||
ASSERT_EQ(1, fetches.size());
|
||||
auto& rtensor = fetches.front().Get<Tensor>();
|
||||
TensorShape expected_shape(expected_dims);
|
||||
ASSERT_EQ(expected_shape, rtensor.Shape());
|
||||
const std::vector<float> found(rtensor.template Data<float>(), rtensor.template Data<float>() + expected_values.size());
|
||||
const std::vector<T> found(rtensor.template Data<T>(), rtensor.template Data<T>() + expected_values.size());
|
||||
ASSERT_EQ(expected_values, found);
|
||||
}
|
||||
|
||||
|
|
@ -217,5 +218,120 @@ TEST(TensorrtExecutionProviderTest, NodeIndexMappingTest) {
|
|||
std::vector<OrtValue> fetche {fetches.back()};
|
||||
VerifyOutputs(fetche, expected_dims_mul_n, expected_values_mul_n);
|
||||
}
|
||||
|
||||
|
||||
TEST(TensorrtExecutionProviderTest, RemoveCycleTest) {
|
||||
onnxruntime::Model model("graph_removecycleTest", false, DefaultLoggingManager().DefaultLogger());
|
||||
auto& graph = model.MainGraph();
|
||||
std::vector<onnxruntime::NodeArg*> inputs;
|
||||
std::vector<onnxruntime::NodeArg*> outputs;
|
||||
|
||||
// FLOAT tensor.
|
||||
ONNX_NAMESPACE::TypeProto float_tensor;
|
||||
float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
|
||||
|
||||
// BOOL tensor.
|
||||
ONNX_NAMESPACE::TypeProto bool_tensor;
|
||||
bool_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_BOOL);
|
||||
bool_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
bool_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
|
||||
bool_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
|
||||
|
||||
// UINT8 tensor.
|
||||
ONNX_NAMESPACE::TypeProto uint8_tensor;
|
||||
uint8_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
|
||||
uint8_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
uint8_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
|
||||
uint8_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
|
||||
|
||||
auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &bool_tensor);
|
||||
auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &bool_tensor);
|
||||
inputs.push_back(&input_arg_1);
|
||||
inputs.push_back(&input_arg_2);
|
||||
auto& output_arg_1 = graph.GetOrCreateNodeArg("xor1_out", &bool_tensor);
|
||||
outputs.push_back(&output_arg_1);
|
||||
graph.AddNode("xor1", "Xor", "node 1.", inputs, outputs);
|
||||
|
||||
inputs.clear();
|
||||
inputs.push_back(&output_arg_1);
|
||||
auto& output_arg_2 = graph.GetOrCreateNodeArg("not_out", &bool_tensor);
|
||||
outputs.clear();
|
||||
outputs.push_back(&output_arg_2);
|
||||
graph.AddNode("not", "Not", "node 2.", inputs, outputs);
|
||||
|
||||
auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &bool_tensor);
|
||||
inputs.clear();
|
||||
inputs.push_back(&output_arg_2);
|
||||
inputs.push_back(&input_arg_3);
|
||||
auto& output_arg_3 = graph.GetOrCreateNodeArg("xor2_out", &bool_tensor);
|
||||
outputs.clear();
|
||||
outputs.push_back(&output_arg_3);
|
||||
graph.AddNode("xor2", "Xor", "node 3.", inputs, outputs);
|
||||
|
||||
inputs.clear();
|
||||
inputs.push_back(&output_arg_2);
|
||||
inputs.push_back(&output_arg_3);
|
||||
auto& output_arg_4 = graph.GetOrCreateNodeArg("M", &bool_tensor);
|
||||
outputs.clear();
|
||||
outputs.push_back(&output_arg_4);
|
||||
graph.AddNode("and", "And", "node 4.", inputs, outputs);
|
||||
|
||||
auto status = graph.Resolve();
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
std::string model_file_name = "trt_execution_provider_removecycle_test.onnx";
|
||||
status = onnxruntime::Model::Save(model, model_file_name);
|
||||
|
||||
std::vector<int64_t> dims_mul_x = {1, 3, 2};
|
||||
std::vector<bool> values_mul_x = {true, false, true, false, true, false};
|
||||
std::vector<int64_t> dims_mul_y = {1, 3, 2};
|
||||
std::vector<bool> values_mul_y = {true, true, false, true, false, false};
|
||||
std::vector<int64_t> dims_mul_z = {1, 3, 2};
|
||||
std::vector<bool> values_mul_z = {true, false, true, false, true, false};
|
||||
|
||||
OrtValue ml_value_x;
|
||||
CreateMLValue<bool>(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_x);
|
||||
OrtValue ml_value_y;
|
||||
CreateMLValue<bool>(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_y, values_mul_y, &ml_value_y);
|
||||
OrtValue ml_value_z;
|
||||
CreateMLValue<bool>(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_y, values_mul_y, &ml_value_z);
|
||||
NameMLValMap feeds;
|
||||
feeds.insert(std::make_pair("X", ml_value_x));
|
||||
feeds.insert(std::make_pair("Y", ml_value_y));
|
||||
feeds.insert(std::make_pair("Z", ml_value_z));
|
||||
|
||||
// prepare outputs
|
||||
std::vector<std::string> output_names;
|
||||
output_names.push_back("M");
|
||||
std::vector<OrtValue> fetches;
|
||||
|
||||
// prepare expected inputs and outputs
|
||||
std::vector<int64_t> expected_dims_mul_m = {1, 3, 2};
|
||||
std::vector<bool> expected_values_mul_m = {false, false, false, false, false, true};
|
||||
|
||||
SessionOptions so;
|
||||
so.session_logid = "TensorrtExecutionProviderTest.RemoveCycleTest";
|
||||
RunOptions run_options;
|
||||
run_options.run_tag = so.session_logid;
|
||||
|
||||
InferenceSession session_object{so};
|
||||
|
||||
TensorrtExecutionProviderInfo epi;
|
||||
epi.device_id = 0;
|
||||
EXPECT_TRUE(session_object.RegisterExecutionProvider(onnxruntime::make_unique<::onnxruntime::TensorrtExecutionProvider>(epi)).IsOK());
|
||||
|
||||
status = session_object.Load(model_file_name);
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
status = session_object.Initialize();
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
|
||||
// Now run
|
||||
status = session_object.Run(run_options, feeds, output_names, &fetches);
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -267,7 +267,7 @@ def check_md5(filename, expected_md5):
|
|||
|
||||
|
||||
|
||||
def setup_test_data(build_dir, configs):
|
||||
def setup_test_data(build_dir, configs):
|
||||
# create a shortcut for test models if there is a 'models' folder in build_dir
|
||||
if is_windows():
|
||||
src_model_dir = os.path.join(build_dir, 'models')
|
||||
|
|
@ -538,10 +538,13 @@ def setup_tensorrt_vars(args):
|
|||
|
||||
# Set maximum number of iterations to detect unsupported nodes and partition the models for TensorRT
|
||||
os.environ["ORT_TENSORRT_MAX_PARTITION_ITERATIONS"] = "1000"
|
||||
|
||||
|
||||
# Set minimum subgraph node size in graph partitioning for TensorRT
|
||||
os.environ["ORT_TENSORRT_MIN_SUBGRAPH_SIZE"] = "1"
|
||||
|
||||
|
||||
# Set FP16 flag
|
||||
os.environ["ORT_TENSORRT_FP16_ENABLE"] = "0"
|
||||
|
||||
return tensorrt_home
|
||||
|
||||
def setup_dml_build(args, cmake_path, build_dir, configs):
|
||||
|
|
@ -670,7 +673,8 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_mult
|
|||
run_subprocess([exe,'-x'] + cmd, cwd=cwd)
|
||||
|
||||
# tensorrt function to run onnx test and model test.
|
||||
def tensorrt_run_onnx_tests(build_dir, configs, onnx_test_data_dir):
|
||||
def tensorrt_run_onnx_tests(args, build_dir, configs, onnx_test_data_dir, provider, num_parallel_models, num_parallel_tests=0):
|
||||
dll_path = os.path.join(args.tensorrt_home, 'lib')
|
||||
for config in configs:
|
||||
cwd = get_config_build_dir(build_dir, config)
|
||||
if is_windows():
|
||||
|
|
@ -679,23 +683,32 @@ def tensorrt_run_onnx_tests(build_dir, configs, onnx_test_data_dir):
|
|||
else:
|
||||
exe = os.path.join(cwd, 'onnx_test_runner')
|
||||
model_dir = os.path.join(build_dir, "models")
|
||||
cmd_base = ['-e', 'tensorrt', '-j', '1']
|
||||
|
||||
cmd_base = []
|
||||
if provider:
|
||||
cmd_base += ["-e", provider]
|
||||
|
||||
if num_parallel_tests != 0:
|
||||
cmd_base += ['-c', str(num_parallel_tests)]
|
||||
|
||||
if num_parallel_models > 0:
|
||||
cmd_base += ["-j", str(num_parallel_models)]
|
||||
|
||||
#onnx test
|
||||
if os.path.exists(onnx_test_data_dir):
|
||||
onnx_test_cmd = cmd_base + [onnx_test_data_dir]
|
||||
run_subprocess([exe] + onnx_test_cmd, cwd=cwd)
|
||||
run_subprocess([exe] + onnx_test_cmd, cwd=cwd, dll_path=dll_path)
|
||||
|
||||
# model test
|
||||
# TensorRT can run most of the model tests, but only part of them is enabled here to save CI build time.
|
||||
if config != 'Debug' and os.path.exists(model_dir):
|
||||
model_dir_opset8 = os.path.join(model_dir, "opset8")
|
||||
model_dir_opset8 = os.path.join(model_dir, "opset8")
|
||||
model_dir_opset8 = glob.glob(os.path.join(model_dir_opset8, "test_*"))
|
||||
model_dir_opset10 = os.path.join(model_dir, "opset10")
|
||||
model_dir_opset10 = glob.glob(os.path.join(model_dir_opset10, "tf_inception_v1"))
|
||||
for dir_path in itertools.chain(model_dir_opset8, model_dir_opset10):
|
||||
model_test_cmd = cmd_base + [dir_path]
|
||||
run_subprocess([exe] + model_test_cmd, cwd=cwd)
|
||||
run_subprocess([exe] + model_test_cmd, cwd=cwd, dll_path=dll_path)
|
||||
|
||||
# dnnl temporary function for running onnx tests and model tests separately.
|
||||
def dnnl_run_onnx_tests(build_dir, configs, onnx_test_data_dir):
|
||||
|
|
@ -713,7 +726,7 @@ def dnnl_run_onnx_tests(build_dir, configs, onnx_test_data_dir):
|
|||
# /data/onnx
|
||||
run_subprocess([exe] + onnxdata_cmd, cwd=cwd)
|
||||
run_subprocess([exe,'-x'] + onnxdata_cmd, cwd=cwd)
|
||||
|
||||
|
||||
if config != 'Debug' and os.path.exists(model_dir):
|
||||
opset7_model_dir = os.path.join(model_dir, 'opset7')
|
||||
opset7_cmd = cmd_base + [opset7_model_dir]
|
||||
|
|
@ -982,10 +995,10 @@ def main():
|
|||
if args.use_tensorrt:
|
||||
# Disable some onnx unit tests that TensorRT doesn't supported yet
|
||||
if not is_windows():
|
||||
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data", "simple")
|
||||
tensorrt_run_onnx_tests(build_dir, configs, onnx_test_data_dir)
|
||||
trt_onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data", "simple")
|
||||
else:
|
||||
tensorrt_run_onnx_tests(build_dir, configs, "")
|
||||
trt_onnx_test_data_dir = ""
|
||||
tensorrt_run_onnx_tests(args, build_dir, configs, trt_onnx_test_data_dir, "tensorrt",1)
|
||||
|
||||
if args.use_cuda and not args.use_tensorrt:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', args.enable_multi_device_test, False, 2)
|
||||
|
|
@ -1000,16 +1013,19 @@ def main():
|
|||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'nuphar', args.enable_multi_device_test, False, 1, 1)
|
||||
|
||||
if args.use_dml:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'dml', args.enable_multi_device_test, False, 1)
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'dml', args.enable_multi_device_test, False, 1)
|
||||
|
||||
# Run some models are disabled to keep memory utilization under control
|
||||
if args.use_dnnl:
|
||||
dnnl_run_onnx_tests(build_dir, configs, onnx_test_data_dir)
|
||||
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, args.enable_multi_device_test, False,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0)
|
||||
|
||||
if args.use_tensorrt:
|
||||
tensorrt_run_onnx_tests(args, build_dir, configs, onnx_test_data_dir, None,1)
|
||||
else:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, args.enable_multi_device_test, False,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0,
|
||||
1 if args.x86 or platform.system() == 'Darwin' else 0)
|
||||
|
||||
# run nuphar python tests last, as it installs ONNX 1.5.0
|
||||
if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar:
|
||||
nuphar_run_python_tests(build_dir, configs)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
pythonInterpreter: '/usr/bin/python3'
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
# The latest TensorRT container (R19.09) only supports ubuntu18.04
|
||||
# Latest TensorRT container only supports ubuntu18.04
|
||||
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d tensorrt -r $(Build.BinariesDirectory) -p 3.6 -x "--build_wheel"'
|
||||
|
||||
displayName: 'Command Line Script'
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ jobs:
|
|||
parameters:
|
||||
AgentPool : 'Linux-Multi-GPU'
|
||||
JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev'
|
||||
# The latest TensorRT container (R19.09) only supports ubuntu18.04
|
||||
# The latest TensorRT container only supports ubuntu18.04
|
||||
BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d tensorrt -r $(Build.BinariesDirectory) -p 3.6 -x "--enable_multi_device_test"'
|
||||
DoNugetPack: 'false'
|
||||
ArtifactName: 'drop-linux'
|
||||
ArtifactName: 'drop-linux'
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
jobs:
|
||||
- job: 'build'
|
||||
pool: 'Win-GPU-CUDA10'
|
||||
pool: 'Win-GPU-2019'
|
||||
strategy:
|
||||
maxParallel: 2
|
||||
matrix:
|
||||
|
|
@ -33,21 +33,8 @@ jobs:
|
|||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: PowerShell@1
|
||||
displayName: 'Set CUDA path'
|
||||
inputs:
|
||||
scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
|
||||
arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion 10.0'
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Download AzCopy (used for download test data script)'
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\azcopy.exe https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe
|
||||
|
||||
- script: |
|
||||
python -m pip install -q pyopenssl setuptools wheel numpy
|
||||
python -m pip install -q pyopenssl setuptools wheel numpy
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
displayName: 'Install python modules'
|
||||
|
||||
|
|
@ -55,7 +42,7 @@ jobs:
|
|||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --numpy_version=1.17.4 --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"'
|
||||
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --msvc_toolset 14.16 --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-7.0.0.11.cuda-10.0.cudnn7.6\TensorRT-7.0.0.11" --cuda_version=10.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
|
|
@ -94,7 +81,7 @@ jobs:
|
|||
del wheel_filename_file
|
||||
python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
|
||||
set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH%
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --test --numpy_version=1.17.4 --build_wheel --enable_onnx_tests --use_tensorrt --cuda_version=10.0 --tensorrt_home="C:\local\TensorRT-6.0.1.5" --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --msvc_toolset 14.16 --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-7.0.0.11.cuda-10.0.cudnn7.6\TensorRT-7.0.0.11" --cuda_version=10.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0
|
||||
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run tests'
|
||||
|
|
@ -103,4 +90,3 @@ jobs:
|
|||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# Tag: nvcr.io/nvidia/tensorrt:19.09-py3
|
||||
# Label: com.nvidia.cuda.version: 10.1.243
|
||||
# Label: com.nvidia.cudnn.version: 7.6.3
|
||||
# Tag: nvcr.io/nvidia/tensorrt:20.01-py3
|
||||
# Label: com.nvidia.cuda.version: 10.2.89
|
||||
# Label: com.nvidia.cudnn.version: 7.6.5
|
||||
# Ubuntu 18.04
|
||||
FROM nvcr.io/nvidia/tensorrt:19.09-py3
|
||||
FROM nvcr.io/nvidia/tensorrt:20.01-py3
|
||||
|
||||
ARG PYTHON_VERSION=3.6
|
||||
|
||||
|
|
|
|||
|
|
@ -69,8 +69,8 @@ else
|
|||
fi
|
||||
docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_DEVICE = "tensorrt" ]; then
|
||||
# TensorRT container release 19.09
|
||||
IMAGE="$BUILD_OS-cuda10.1-cudnn7.6-tensorrt6.0"
|
||||
# TensorRT container release 20.01
|
||||
IMAGE="$BUILD_OS-cuda10.2-cudnn7.6-tensorrt7.0"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_tensorrt
|
||||
docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_DEVICE = "openvino" ]; then
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
set PATH=%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda\bin;C:\local\cuda_10.0.130_win10;%PATH%;C:\local\systools
|
||||
set PATH=%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;C:\local\cudnn-10.0-windows10-x64-v7.6.5.32\cuda\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0;%PATH%;C:\local\systools
|
||||
Loading…
Reference in a new issue