mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Update TensorRT to version 6.0.1.5 (#1966)
* remove onnx-tensorrt submodule * add new onnx-tensorrt submodule (experiment) for trt6 * update engine build for trt6 * update compile and compute for tensorrt6.0 * Update tensorrt_execution_provider.cc * Update tensorrt_execution_provider.cc * Update tensorrt_execution_provider.cc * Update tensorrt_execution_provider.cc * switch to onnx-tensorrt master for TensorRT6' * Update tensorrt_execution_provider.cc * Handle dynamic batch size and add memcpy in TensorRT EP * update test cases * Update tensorrt_execution_provider.cc * update onnx-tensorrt submodule * Update Dockerfile.ubuntu_tensorrt * Update Dockerfile.ubuntu_tensorrt * Update run_dockerbuild.sh * Update run_dockerbuild.sh * Update install_ubuntu.sh * Update concat_op_test.cc * Update tensorrt_execution_provider.cc * Upgrade TensorRT to version 6.0.1.5 * Update onnxruntime_providers.cmake * Update CMakeLists.txt * Update reduction_ops_test.cc * Update install_ubuntu.sh * Update Dockerfile.ubuntu_tensorrt * Update Dockerfile.tensorrt * Update BUILD.md * Update run_dockerbuild.sh * Update install_ubuntu.sh * Update onnxruntime_providers.cmake * Update install_ubuntu.sh * Update install_ubuntu.sh * Update gemm_test.cc * Update gather_op_test.cc * Update CMakeLists.txt * Removed submodule * update onnx-tensorrt submodule * Add Ubuntu18.04 build option * Add Ubuntu18.04 build option * Add Ubuntu18.04 build option * Add Ubuntu18.04 build option * Remove redundency * Fix issue that it does not add memcopy node correctly if some nodes fall back to CUDA EP. e.g. after partition, there's TRT_Node -> Cuda_node (with CPU memory expected), we still need to add memcpy node between them. * update for Trt Windows build * Update onnxruntime_providers.cmake * Disable opset11 tests on TensorRT * Update pad_test.cc * Update build.py * update scripts for ubuntu18.04 * Disable warning for Windows build
This commit is contained in:
stevenlix
George Wu
parent
4bb6385dca
commit
544e53e24e
28 changed files with 321 additions and 142 deletions
7
.gitmodules
vendored
7
.gitmodules
vendored
|
|
@ -34,13 +34,12 @@
|
|||
[submodule "cmake/external/spdlog"]
|
||||
path = cmake/external/spdlog
|
||||
url = https://github.com/gabime/spdlog.git
|
||||
[submodule "cmake/external/onnx-tensorrt"]
|
||||
path = cmake/external/onnx-tensorrt
|
||||
url = https://github.com/onnx/onnx-tensorrt.git
|
||||
branch = 5.1
|
||||
[submodule "cmake/external/mimalloc"]
|
||||
path = cmake/external/mimalloc
|
||||
url = https://github.com/microsoft/mimalloc.git
|
||||
[submodule "cmake/external/cub"]
|
||||
path = cmake/external/cub
|
||||
url = https://github.com/NVlabs/cub.git
|
||||
[submodule "cmake/external/onnx-tensorrt"]
|
||||
path = cmake/external/onnx-tensorrt
|
||||
url = https://github.com/onnx/onnx-tensorrt.git
|
||||
|
|
|
|||
2
BUILD.md
2
BUILD.md
|
|
@ -187,7 +187,7 @@ If you want to build with an earlier version, you must temporarily remove the 'C
|
|||
### TensorRT
|
||||
ONNX Runtime supports the TensorRT execution provider (released as preview). You will need to download and install [CUDA](https://developer.nvidia.com/cuda-toolkit), [cuDNN](https://developer.nvidia.com/cudnn) and [TensorRT](https://developer.nvidia.com/nvidia-tensorrt-download).
|
||||
|
||||
The TensorRT execution provider for ONNX Runtime is built and tested with CUDA 9.0/CUDA 10.0, cuDNN 7.1 and TensorRT 5.0.2.6.
|
||||
The TensorRT execution provider for ONNX Runtime is built and tested with CUDA 10.1, cuDNN 7.6 and TensorRT 6.0.1.5.
|
||||
|
||||
- The path to the CUDA installation must be provided via the CUDA_PATH environment variable, or the `--cuda_home parameter`. The CUDA path should contain `bin`, `include` and `lib` directories.
|
||||
- The path to the CUDA `bin` directory must be added to the PATH environment variable so that `nvcc` is found.
|
||||
|
|
|
|||
|
|
@ -615,6 +615,8 @@ endif()
|
|||
if (onnxruntime_USE_TENSORRT)
|
||||
if (WIN32)
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DELAYLOAD:nvinfer.dll")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
|
|
|||
2
cmake/external/onnx-tensorrt
vendored
2
cmake/external/onnx-tensorrt
vendored
|
|
@ -1 +1 @@
|
|||
Subproject commit 6c37109733a9bbf8211f0ca78a85804cb376eca0
|
||||
Subproject commit bba4dee184cc03d6fd5086c90d974537e72eba23
|
||||
|
|
@ -278,18 +278,19 @@ if (onnxruntime_USE_TENSORRT)
|
|||
set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
|
||||
if (WIN32)
|
||||
set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd2220")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /wd4324 /wd4701")
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4701 /wd4805")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4805")
|
||||
endif()
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -include algorithm")
|
||||
set(PROTOBUF_LIBRARY libprotobuf)
|
||||
set(DISABLED_WARNINGS_FOR_TRT /wd4267 /wd4244 /wd4996)
|
||||
list(APPEND CUDA_LIBRARIES cudart.lib cudadevrt.lib)
|
||||
endif()
|
||||
if ( CMAKE_COMPILER_IS_GNUCC )
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-missing-field-initializers")
|
||||
endif()
|
||||
set(CXX_VERSION_DEFINED TRUE)
|
||||
set(CXX_VERSION_DEFINED TRUE)
|
||||
add_subdirectory(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt)
|
||||
set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS})
|
||||
if (WIN32)
|
||||
|
|
@ -298,7 +299,6 @@ if (onnxruntime_USE_TENSORRT)
|
|||
unset(OLD_CMAKE_CXX_FLAGS)
|
||||
unset(OLD_CMAKE_CUDA_FLAGS)
|
||||
set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199")
|
||||
set_target_properties(nvonnxparser_runtime PROPERTIES LINK_FLAGS "/ignore:4199")
|
||||
set_target_properties(trt_onnxify PROPERTIES LINK_FLAGS "/ignore:4199")
|
||||
target_compile_definitions(trt_onnxify PRIVATE ONNXIFI_BUILD_LIBRARY=1)
|
||||
target_sources(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc)
|
||||
|
|
@ -314,7 +314,7 @@ if (onnxruntime_USE_TENSORRT)
|
|||
include_directories(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt)
|
||||
include_directories(${TENSORRT_INCLUDE_DIR})
|
||||
set(trt_link_libs cudnn ${CMAKE_DL_LIBS} ${TENSORRT_LIBRARY})
|
||||
set(onnxparser_link_libs nvonnxparser_static nvonnxparser_plugin)
|
||||
set(onnxparser_link_libs nvonnxparser_static)
|
||||
|
||||
file(GLOB_RECURSE onnxruntime_providers_tensorrt_cc_srcs CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/core/providers/tensorrt/*.h"
|
||||
|
|
@ -464,7 +464,7 @@ if (onnxruntime_USE_NUPHAR)
|
|||
add_dependencies(onnxruntime_providers_nuphar ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/nuphar DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
endif()
|
||||
|
||||
|
||||
if (onnxruntime_USE_NNAPI)
|
||||
add_definitions(-DUSE_NNAPI=1)
|
||||
option(DNN_READ_ONNX "" ON)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
# Dockerfile to run ONNXRuntime with TensorRT integration
|
||||
|
||||
# nVidia TensorRT Base Image
|
||||
FROM nvcr.io/nvidia/tensorrt:19.06-py3
|
||||
FROM nvcr.io/nvidia/tensorrt:19.09-py3
|
||||
MAINTAINER Vinitra Swamy "viswamy@microsoft.com"
|
||||
|
||||
ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr
|
|||
TransformerMemcpyImpl copy_impl(graph, provider);
|
||||
auto current_modified = copy_impl.ModifyGraph(registry_manager_);
|
||||
modified = modified || current_modified;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -166,9 +167,8 @@ bool TransformerMemcpyImpl::ModifyGraph(const KernelRegistryManager& kernel_regi
|
|||
}
|
||||
|
||||
void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelRegistryManager& kernel_registries, InitializedTensorSet& initializers_consumed) {
|
||||
if (node.GetExecutionProviderType() == provider_
|
||||
|| (node.GetExecutionProviderType() == kCudaExecutionProvider && provider_ == kTensorrtExecutionProvider)
|
||||
|| (node.GetExecutionProviderType() == kTensorrtExecutionProvider && provider_ == kCudaExecutionProvider)) {
|
||||
auto node_provider_type = node.GetExecutionProviderType();
|
||||
if ((node_provider_type == provider_) || (node_provider_type == kCudaExecutionProvider && kTensorrtExecutionProvider == provider_)) {
|
||||
provider_nodes_.insert(&node);
|
||||
// note KernelCreateInfo might be nullptr for custom kernel
|
||||
const KernelCreateInfo* kci = nullptr;
|
||||
|
|
@ -205,11 +205,11 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg
|
|||
else
|
||||
provider_output_defs_.insert(arg);
|
||||
}
|
||||
} else {
|
||||
} else if (node_provider_type != kCudaExecutionProvider && node_provider_type != kTensorrtExecutionProvider) {
|
||||
// TODO: copy between devices? i.e. multiple GPUs
|
||||
if (node.GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider &&
|
||||
node.GetExecutionProviderType() != onnxruntime::kNGraphExecutionProvider && !node.GetExecutionProviderType().empty()) {
|
||||
ORT_THROW("Execution type '", node.GetExecutionProviderType(), "' doesn't support memcpy ");
|
||||
if (node_provider_type != onnxruntime::kCpuExecutionProvider &&
|
||||
node_provider_type != onnxruntime::kNGraphExecutionProvider && !node_provider_type.empty()) {
|
||||
ORT_THROW("Execution type '", node_provider_type, "' doesn't support memcpy ");
|
||||
}
|
||||
|
||||
for (const auto* arg : node.InputDefs()) {
|
||||
|
|
@ -243,7 +243,8 @@ void TransformerMemcpyImpl::BuildDefsMapping(const onnxruntime::NodeArg* arg, co
|
|||
output_it != it.MutableOutputDefs().end() ? static_cast<int>(output_it - it.MutableOutputDefs().begin()) : -1;
|
||||
if (arg_input_index == -1 && arg_output_index == -1)
|
||||
continue;
|
||||
if (it.GetExecutionProviderType() == provider_) {
|
||||
auto node_provider_type = it.GetExecutionProviderType();
|
||||
if ((node_provider_type == provider_) || (node_provider_type == kCudaExecutionProvider && kTensorrtExecutionProvider == provider_)) {
|
||||
const KernelCreateInfo* kci = nullptr;
|
||||
kernel_registries.SearchKernelRegistry(it, &kci);
|
||||
if (arg_input_index != -1) {
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@
|
|||
|
||||
#include "tensorrt_execution_provider.h"
|
||||
#include "core/providers/cuda/cuda_allocator.h"
|
||||
#include "core/providers/cuda/math/unary_elementwise_ops_impl.h"
|
||||
#include "core/session/onnxruntime_cxx_api.h"
|
||||
#include "core/framework/execution_provider.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/framework/kernel_registry.h"
|
||||
#include "core/framework/compute_capability.h"
|
||||
#include "core/framework/memcpy.h"
|
||||
#include "core/providers/cpu/cpu_execution_provider.h"
|
||||
#include "core/providers/cuda/cuda_common.h"
|
||||
#include "core/providers/cuda/cuda_fence.h"
|
||||
|
|
@ -17,7 +19,6 @@
|
|||
#include "cuda_runtime_api.h"
|
||||
#include "gsl/gsl"
|
||||
#include "core/graph/model.h"
|
||||
#include "cuda_runtime_api.h"
|
||||
#include "core/providers/cuda/gpu_data_transfer.h"
|
||||
|
||||
using namespace std;
|
||||
|
|
@ -26,12 +27,6 @@ using namespace ::onnxruntime::logging;
|
|||
|
||||
namespace onnxruntime {
|
||||
|
||||
// Per TensorRT documentation, logger needs to be a singleton.
|
||||
TensorrtLogger& GetTensorrtLogger() {
|
||||
static TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
|
||||
return trt_logger;
|
||||
}
|
||||
|
||||
#define CHECK_CUDA(call) \
|
||||
do { \
|
||||
cudaError_t status = call; \
|
||||
|
|
@ -40,6 +35,60 @@ TensorrtLogger& GetTensorrtLogger() {
|
|||
} \
|
||||
} while (0)
|
||||
|
||||
ONNX_OPERATOR_KERNEL_EX(
|
||||
MemcpyFromHost,
|
||||
kOnnxDomain,
|
||||
1,
|
||||
kTensorrtExecutionProvider,
|
||||
KernelDefBuilder()
|
||||
.InputMemoryType<OrtMemTypeCPUInput>(0)
|
||||
.ExecQueueId(kCudaStreamCopyIn)
|
||||
.TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()),
|
||||
Memcpy);
|
||||
|
||||
ONNX_OPERATOR_KERNEL_EX(
|
||||
MemcpyToHost,
|
||||
kOnnxDomain,
|
||||
1,
|
||||
kTensorrtExecutionProvider,
|
||||
KernelDefBuilder()
|
||||
.OutputMemoryType<OrtMemTypeCPUOutput>(0)
|
||||
.ExecQueueId(kCudaStreamCopyOut)
|
||||
.TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()),
|
||||
Memcpy);
|
||||
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kTensorrtExecutionProvider, kOnnxDomain, 1, MemcpyFromHost);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kTensorrtExecutionProvider, kOnnxDomain, 1, MemcpyToHost);
|
||||
|
||||
static void RegisterTensorrtKernels(KernelRegistry& kernel_registry) {
|
||||
static const BuildKernelCreateInfoFn function_table[] = {
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kTensorrtExecutionProvider, kOnnxDomain, 1, MemcpyFromHost)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kTensorrtExecutionProvider, kOnnxDomain, 1, MemcpyToHost)>,
|
||||
};
|
||||
|
||||
for (auto& function_table_entry : function_table) {
|
||||
kernel_registry.Register(function_table_entry());
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<KernelRegistry> GetTensorrtKernelRegistry() {
|
||||
std::shared_ptr<KernelRegistry> kernel_registry = std::make_shared<KernelRegistry>();
|
||||
RegisterTensorrtKernels(*kernel_registry);
|
||||
|
||||
return kernel_registry;
|
||||
}
|
||||
|
||||
std::shared_ptr<KernelRegistry> TensorrtExecutionProvider::GetKernelRegistry() const {
|
||||
static std::shared_ptr<KernelRegistry> kernel_registry = onnxruntime::GetTensorrtKernelRegistry();
|
||||
return kernel_registry;
|
||||
}
|
||||
|
||||
// Per TensorRT documentation, logger needs to be a singleton.
|
||||
TensorrtLogger& GetTensorrtLogger() {
|
||||
static TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
|
||||
return trt_logger;
|
||||
}
|
||||
|
||||
TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProviderInfo& info)
|
||||
: IExecutionProvider{onnxruntime::kTensorrtExecutionProvider}, device_id_(info.device_id) {
|
||||
CUDA_CALL_THROW(cudaSetDevice(device_id_));
|
||||
|
|
@ -210,7 +259,9 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
|
|||
SubGraphCollection_t parser_nodes_list;
|
||||
TensorrtLogger& trt_logger = GetTensorrtLogger();
|
||||
auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
|
||||
const auto explicitBatch = 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
|
||||
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
|
||||
auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
|
||||
trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_list);
|
||||
|
||||
|
|
@ -268,7 +319,8 @@ TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
|
|||
SubGraphCollection_t parser_nodes_vector;
|
||||
TensorrtLogger& trt_logger = GetTensorrtLogger();
|
||||
auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
|
||||
const auto explicitBatch = 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
|
||||
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
|
||||
trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_vector);
|
||||
|
||||
|
|
@ -336,7 +388,9 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
// Create TensorRT engine
|
||||
TensorrtLogger& trt_logger = GetTensorrtLogger();
|
||||
auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
|
||||
const auto explicitBatch = 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
|
||||
auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
|
||||
auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
|
||||
trt_parser->parse(string_buf.data(), string_buf.size());
|
||||
|
||||
|
|
@ -353,8 +407,35 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
}
|
||||
|
||||
trt_builder->setMaxBatchSize(max_batch_size_);
|
||||
trt_builder->setMaxWorkspaceSize(max_workspace_size_);
|
||||
auto trt_engine = unique_pointer<nvinfer1::ICudaEngine>(trt_builder->buildCudaEngine(*trt_network.get()));
|
||||
auto trt_config = unique_pointer<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
|
||||
trt_config->setMaxWorkspaceSize(max_workspace_size_);
|
||||
|
||||
//Set optimization profile for dynamic shapes
|
||||
//Only support dynamic batch size on the first dimension for now
|
||||
//TODO: add full dynamic shape support
|
||||
auto trt_profile = trt_builder->createOptimizationProfile();
|
||||
bool dynamic_shape = false;
|
||||
for (unsigned int i = 0, n = trt_network->getNbInputs(); i < n; i++) {
|
||||
auto input = trt_network->getInput(i);
|
||||
nvinfer1::Dims dims = input->getDimensions();
|
||||
nvinfer1::Dims dims_min = dims;
|
||||
nvinfer1::Dims dims_opt = dims;
|
||||
nvinfer1::Dims dims_max = dims;
|
||||
if (dims.d[0] == -1) {
|
||||
dims_min.d[0] = 1;
|
||||
dims_opt.d[0] = max_batch_size_;
|
||||
dims_max.d[0] = max_batch_size_;
|
||||
trt_profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kMIN, dims_min);
|
||||
trt_profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kOPT, dims_opt);
|
||||
trt_profile->setDimensions(input->getName(), nvinfer1::OptProfileSelector::kMAX, dims_max);
|
||||
dynamic_shape = true;
|
||||
}
|
||||
}
|
||||
if (dynamic_shape) {
|
||||
trt_config->addOptimizationProfile(trt_profile);
|
||||
}
|
||||
|
||||
auto trt_engine = unique_pointer<nvinfer1::ICudaEngine>(trt_builder->buildEngineWithConfig(*trt_network, *trt_config));
|
||||
ORT_ENFORCE(trt_engine != nullptr);
|
||||
|
||||
// Build TensorRT context
|
||||
|
|
@ -405,11 +486,6 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
const auto& graph_output = model_proto.graph().output();
|
||||
const auto& tensor_type = graph_output[i].type().tensor_type();
|
||||
output_types[bindingIndex] = tensor_type.elem_type();
|
||||
|
||||
const auto& tensor_shape = tensor_type.shape();
|
||||
if (tensor_shape.dim_size() == 1 && output_shapes[bindingIndex].back() == 1) {
|
||||
output_shapes[bindingIndex].pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
ORT_ENFORCE(trt_engine->getNbBindings() == (num_inputs + num_outputs));
|
||||
|
|
@ -449,51 +525,79 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
const std::vector<int>& input_indexes = (trt_state->input_info)[0];
|
||||
const std::vector<int>& output_indexes = (trt_state->output_info)[0];
|
||||
const std::vector<int>& output_types = (trt_state->output_info)[2];
|
||||
std::vector<std::vector<int64_t>> output_shapes = trt_state->output_shapes;
|
||||
|
||||
int num_binding_inputs = input_indexes.size();
|
||||
int num_binding_outputs = output_indexes.size();
|
||||
int total_bindings = num_binding_inputs + num_binding_outputs;
|
||||
std::vector<void*> buffers(total_bindings);
|
||||
int batch_size = 1;
|
||||
|
||||
bool dynamic_shape = false;
|
||||
if (!trt_state->context->allInputDimensionsSpecified()) {
|
||||
dynamic_shape = true;
|
||||
}
|
||||
|
||||
// Get batch size and allocate cuda memory for inputs
|
||||
for (int i = 0, end = num_binding_inputs; i < end; ++i) {
|
||||
const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_indexes[i]);
|
||||
auto tensor_info = ort.GetTensorTypeAndShape(input_tensor);
|
||||
const auto& tensor_shape = ort.GetTensorShape(tensor_info);
|
||||
|
||||
//Set dynamic shapes
|
||||
nvinfer1::Dims dimensions = trt_state->context->getEngine().getBindingDimensions(static_cast<int>(i));
|
||||
if (dynamic_shape) {
|
||||
for (int j = 0, end = tensor_shape.size(); j < end; ++j)
|
||||
dimensions.d[j] = tensor_shape[j];
|
||||
trt_state->context->setBindingDimensions(i, dimensions);
|
||||
}
|
||||
|
||||
auto tensor_type = ort.GetTensorElementType(tensor_info);
|
||||
ort.ReleaseTensorTypeAndShapeInfo(tensor_info);
|
||||
|
||||
const int input_batch_size = tensor_shape[0];
|
||||
if (i > 0 && batch_size != input_batch_size) {
|
||||
ORT_THROW("Input batch size is inconsistent");
|
||||
}
|
||||
batch_size = input_batch_size;
|
||||
|
||||
if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
|
||||
buffers[i] = const_cast<float*>(ort.GetTensorData<float>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
|
||||
buffers[i] = const_cast<int8_t*>(ort.GetTensorData<int8_t>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
buffers[i] = const_cast<int32_t*>(ort.GetTensorData<int32_t>(input_tensor));
|
||||
} else if (tensor_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) {
|
||||
// Cast INT64 input to INT32 because TensorRT doesn't fully support INT64
|
||||
int input_dim_size = 1;
|
||||
for (int j = 0, end = dimensions.nbDims; j < end; ++j) {
|
||||
input_dim_size *= tensor_shape[j];
|
||||
}
|
||||
CHECK_CUDA(cudaMalloc(&buffers[i], input_dim_size * sizeof(int32_t)));
|
||||
cuda::Impl_Cast<int64_t, int32_t>(ort.GetTensorData<int64_t>(input_tensor), reinterpret_cast<int32_t*>(buffers[i]), input_dim_size);
|
||||
} else {
|
||||
return common::Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED);
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate cuda memory for outputs
|
||||
// Allocate CUDA memory for outputs
|
||||
std::vector<int> output_dim_size(num_binding_outputs, 1);
|
||||
std::vector<OrtValue*> output_tensor(num_binding_outputs, nullptr);
|
||||
for (int i = 0, end = num_binding_outputs; i < end; ++i) {
|
||||
int output_index = output_indexes[i];
|
||||
output_shapes[i].insert(output_shapes[i].begin(), batch_size);
|
||||
// Set dynamic shapes
|
||||
nvinfer1::Dims dimensions = trt_state->context->getBindingDimensions(static_cast<int>(i + num_binding_inputs));
|
||||
for (int j = 0, end = trt_state->output_shapes[i].size(); j < end; ++j) {
|
||||
trt_state->output_shapes[i][j] = dimensions.d[j];
|
||||
}
|
||||
|
||||
int output_index = output_indexes[i];
|
||||
output_tensor[i] = ort.KernelContext_GetOutput(context, output_index, trt_state->output_shapes[i].data(), trt_state->output_shapes[i].size());
|
||||
|
||||
OrtValue* output_tensor = ort.KernelContext_GetOutput(context, output_index, output_shapes[i].data(), output_shapes[i].size());
|
||||
if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<float>(output_tensor);
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<float>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<int8_t>(output_tensor);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 || output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<int32_t>(output_tensor);
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<int8_t>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
buffers[i + num_binding_inputs] = ort.GetTensorMutableData<int32_t>(output_tensor[i]);
|
||||
} else if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) {
|
||||
// Allocate INT32 CUDA memory for INT64 output type because TensorRT doesn't fully support INT64
|
||||
for (int j = 0, end = dimensions.nbDims; j < end; ++j) {
|
||||
output_dim_size[i] *= dimensions.d[j];
|
||||
}
|
||||
CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], output_dim_size[i] * sizeof(int32_t)));
|
||||
|
||||
} else {
|
||||
return common::Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
|
@ -501,13 +605,13 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
|
||||
// Run TRT inference
|
||||
std::lock_guard<OrtMutex> lock(*(trt_state->tensorrt_mu_ptr));
|
||||
bool ret = trt_state->context->enqueue(batch_size, &buffers[0], nullptr, nullptr);
|
||||
if (!ret) {
|
||||
if (trt_state->context->getEngine().getMaxBatchSize() < batch_size) {
|
||||
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
|
||||
"TRT enqueue failed: Set ORT_TENSORRT_MAX_BATCH_SIZE environment variable to at least " + to_string(batch_size));
|
||||
trt_state->context->enqueueV2(&buffers[0], nullptr, nullptr);
|
||||
|
||||
// Cast INT64 input to INT32 because TensorRT doesn't fully support INT64
|
||||
for (int i = 0, end = num_binding_outputs; i < end; ++i) {
|
||||
if (output_types[i] == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) {
|
||||
cuda::Impl_Cast<int32_t, int64_t>(reinterpret_cast<int32_t*>(buffers[i + num_binding_inputs]), ort.GetTensorMutableData<int64_t>(output_tensor[i]), output_dim_size[i]);
|
||||
}
|
||||
return common::Status(common::ONNXRUNTIME, common::FAIL, "Failed to enqueue to TRT execution context.");
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
@ -519,3 +623,4 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime:
|
|||
return Status::OK();
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
explicit TensorrtExecutionProvider(const TensorrtExecutionProviderInfo& info);
|
||||
virtual ~TensorrtExecutionProvider();
|
||||
|
||||
virtual std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
|
||||
std::unique_ptr<onnxruntime::IDataTransfer> GetDataTransfer() const override;
|
||||
|
||||
std::vector<std::unique_ptr<ComputeCapability>>
|
||||
|
|
|
|||
|
|
@ -57,8 +57,8 @@ TEST(MathOpTest, Clip) {
|
|||
-5.0f, 0.0f, 5.0f,
|
||||
-5.0f, 2.0f, 5.0f});
|
||||
|
||||
// nGraph does not support Clip opset 11 yet.
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider});
|
||||
// nGraph and Tensorrt does not support Clip opset 11 yet.
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
|
|
|||
|
|
@ -401,7 +401,7 @@ TEST(MathOpTest, Abs_int8) {
|
|||
std::vector<int64_t> dims{4};
|
||||
test.AddInput<int8_t>("X", dims, {1, 2, -1, -5});
|
||||
test.AddOutput<int8_t>("Y", dims, {1, 2, 1, 5});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: INT8, Assertion `regionRanges != nullptr' failed
|
||||
}
|
||||
|
||||
TEST(MathOpTest, Abs_int32) {
|
||||
|
|
@ -429,7 +429,7 @@ TEST(MathOpTest, Neg_int8) {
|
|||
std::vector<int64_t> dims{4};
|
||||
test.AddInput<int8_t>("X", dims, {1, -2, 0, -10});
|
||||
test.AddOutput<int8_t>("Y", dims, {-1, 2, 0, 10});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: INT8 is not supported
|
||||
}
|
||||
|
||||
TEST(MathOpTest, Neg_int32) {
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) {
|
|||
test.AddInput<MLFloat16>("B", {4, 3}, f_B);
|
||||
test.AddInput<MLFloat16>("C", {2, 3}, f_C);
|
||||
test.AddOutput<MLFloat16>("Y", {2, 3}, f_Y);
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: fp16 is not supported
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -116,7 +116,7 @@ TEST(GemmOpTest, GemmAlphaBeta) {
|
|||
test.AddOutput<float>("Y", {2, 3},
|
||||
{7.0f, 7.0f, 7.0f,
|
||||
-3.0f, -3.0f, -3.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser
|
||||
}
|
||||
|
||||
TEST(GemmOpTest, GemmNaN) {
|
||||
|
|
@ -135,7 +135,7 @@ TEST(GemmOpTest, GemmNaN) {
|
|||
test.AddOutput<float>("Y", {2, 3},
|
||||
{10.0f, 10.0f, 10.0f,
|
||||
-10.0f, -10.0f, -10.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser
|
||||
}
|
||||
|
||||
TEST(GemmOpTest, GemmScalarBroadcast) {
|
||||
|
|
@ -229,7 +229,7 @@ TEST(GemmOpTest, GemmEmptyTensor) {
|
|||
test.AddInput<float>("C", {3}, std::vector<float>(3, 1.0f));
|
||||
test.AddOutput<float>("Y", {0, 3},
|
||||
{});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: doesn't support dynamic shape yet
|
||||
}
|
||||
|
||||
TEST(GemmOpTest, GemmNoBiasOpset11) {
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ TEST(ReductionOpTest, ReduceL1_do_not_keepdims) {
|
|||
9.0f, 10.0f,
|
||||
11.0f, 12.0f});
|
||||
test.AddOutput<float>("reduced", {3, 2}, {3.0f, 7.0f, 11.0f, 15.0f, 19.0f, 23.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceL1_do_not_keepdims_2) {
|
||||
|
|
@ -114,7 +114,7 @@ TEST(ReductionOpTest, ReduceL1_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {6.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceL1_keepdims) {
|
||||
|
|
@ -196,7 +196,7 @@ TEST(ReductionOpTest, ReduceL2_do_not_keepdims) {
|
|||
9.0f, 10.0f,
|
||||
11.0f, 12.0f});
|
||||
test.AddOutput<float>("reduced", {3, 2}, {2.23606798f, 5.0f, 7.81024968f, 10.63014581f, 13.45362405f, 16.2788206f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceL2_do_not_keepdims_2) {
|
||||
|
|
@ -206,7 +206,7 @@ TEST(ReductionOpTest, ReduceL2_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {3.741657387f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceL2_keepdims) {
|
||||
|
|
@ -257,7 +257,7 @@ TEST(ReductionOpTest, ReduceL2_int32) {
|
|||
9, 10,
|
||||
11, 12});
|
||||
test.AddOutput<int32_t>("reduced", {2}, {15, 20});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Int32 not allowed as input to this layer
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceLogSum) {
|
||||
|
|
@ -298,7 +298,7 @@ TEST(ReductionOpTest, ReduceLogSum_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {1.79175947f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceLogSumExp_default_axes_keepdims) {
|
||||
|
|
@ -331,7 +331,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims) {
|
|||
55.0f, 1.0f,
|
||||
60.0f, 2.0f});
|
||||
test.AddOutput<float>("reduced", {3, 2}, {20.0f, 2.31326175f, 40.00004578f, 2.31326175f, 60.00671387f, 2.31326175f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims_2) {
|
||||
|
|
@ -341,7 +341,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {3.40760596f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceLogSumExp_keepdims) {
|
||||
|
|
@ -435,7 +435,7 @@ TEST(ReductionOpTest, ReduceMax_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{5.0f, 1.0f, 20.0f});
|
||||
test.AddOutput<float>("reduced", {}, {20.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceMax_keepdims) {
|
||||
|
|
@ -546,7 +546,7 @@ TEST(ReductionOpTest, ReduceMean_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {2.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceMean_keepdims) {
|
||||
|
|
@ -640,7 +640,7 @@ TEST(ReductionOpTest, ReduceMin_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{5.0f, 1.0f, 20.0f});
|
||||
test.AddOutput<float>("reduced", {}, {1.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceMin_keepdims) {
|
||||
|
|
@ -830,7 +830,7 @@ TEST(ReductionOpTest, ReduceSum_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {6.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceSum_keepdims) {
|
||||
|
|
@ -898,7 +898,7 @@ TEST(ReductionOpTest, ReduceSumSquare_int32) {
|
|||
9, 10,
|
||||
11, 12});
|
||||
test.AddOutput<int32_t>("reduced", {1, 2, 1}, {247, 403});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: Int32 not allowed as input to this layer
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceSumSquare_default_axes_keepdims) {
|
||||
|
|
@ -941,7 +941,7 @@ TEST(ReductionOpTest, ReduceSumSquare_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {14.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceSumSquare_keepdims) {
|
||||
|
|
@ -1000,7 +1000,7 @@ TEST(ReductionOpTest, ReduceProd_do_not_keepdims_2) {
|
|||
test.AddInput<float>("data", {3},
|
||||
{1.0f, 2.0f, 3.0f});
|
||||
test.AddOutput<float>("reduced", {}, {6.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});//TensorRT: full reduce without keepDimensions is not supported with explicit batch
|
||||
}
|
||||
|
||||
TEST(ReductionOpTest, ReduceProd_keepdims) {
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ TEST(ConcatOpTest, Concat1D_2) {
|
|||
test.AddInput<float>("input2", {2}, {2.0f, 3.0f});
|
||||
test.AddInput<float>("input3", {0}, {});
|
||||
test.AddOutput<float>("concat_result", {3}, {1.0f, 2.0f, 3.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: no support for dynamic shape tensor
|
||||
}
|
||||
|
||||
TEST(ConcatOpTest, Concat2D_1) {
|
||||
|
|
@ -104,7 +104,7 @@ TEST(ConcatOpTest, Concat2D_3) {
|
|||
test.AddInput<float>("input2", {1, 0}, {});
|
||||
test.AddInput<float>("input3", {1, 0}, {});
|
||||
test.AddOutput<float>("concat_result", {1, 0}, {});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: no support for dynamic shape tensor
|
||||
}
|
||||
|
||||
TEST(ConcatOpTest, Concat3D_1) {
|
||||
|
|
|
|||
|
|
@ -254,7 +254,7 @@ TEST(GatherOpTest, Gather_axis1_indices2d_int8) {
|
|||
{1, 0, 2, 1,
|
||||
11, 10, 12, 11,
|
||||
21, 20, 22, 21});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Assertion `regionRanges != nullptr' failed
|
||||
}
|
||||
|
||||
TEST(GatherOpTest, Gather_axis1_indices2d_string) {
|
||||
|
|
@ -320,7 +320,7 @@ TEST(GatherOpTest, Gather_axis1_neg_indices2d_int8) {
|
|||
{1, 0, 2, 1,
|
||||
11, 10, 12, 11,
|
||||
21, 20, 22, 21});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Assertion `regionRanges != nullptr' failed
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
|
|
|||
|
|
@ -31,8 +31,8 @@ static void RunTest(
|
|||
test2.AddInput<int64_t>("pads", {static_cast<int64_t>(pads.size())}, pads);
|
||||
test2.AddInput<float>("value", {1}, {value});
|
||||
test2.AddOutput<float>("output", output_dims, output);
|
||||
// NGraph does not yet support opset-11 and builds break on this test, hence exclude the EP
|
||||
test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider});
|
||||
// NGraph and TensorRT do not yet support opset-11 and builds break on this test, hence exclude the EP
|
||||
test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kTensorrtExecutionProvider});
|
||||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
|
||||
|
|
@ -43,7 +43,8 @@ static void RunTest(
|
|||
test3.AddInput<int64_t>("pads", {static_cast<int64_t>(pads.size())}, pads);
|
||||
test3.AddInput<float>("value", {1}, {value});
|
||||
test3.AddOutput<float>("output", output_dims, output);
|
||||
test3.Run();
|
||||
//TensorRT does not support pads as an input
|
||||
test3.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
|
||||
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ TEST(TensorOpTest, Reshape) {
|
|||
test.AddInput<float>("data", {2, 3}, std::vector<float>(6, 1.0f));
|
||||
test.AddInput<int64_t>("shape", {3}, {-1, 0, 2});
|
||||
test.AddOutput<float>("reshaped", {1, 3, 2}, std::vector<float>(6, 1.0f));
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNupharExecutionProvider}); // Nuphar only supports reshape shape from initializer
|
||||
//TensorRT doesn't support dynamic shape tensor for now
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNupharExecutionProvider, kTensorrtExecutionProvider}); // Nuphar only supports reshape shape from initializer
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, ReshapeWithEmptyDim) {
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ TEST(TensorOpTest, Unsqueeze_Duplicate) {
|
|||
test.AddAttribute("axes", std::vector<int64_t>{2, 1, 0, 2});
|
||||
test.AddInput<float>("input", {2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
|
||||
test.AddOutput<float>("output", {1, 1, 1, 2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
|
||||
test.Run(OpTester::ExpectResult::kExpectFailure, "'axes' has a duplicate axis");
|
||||
test.Run(OpTester::ExpectResult::kExpectFailure, "'axes' has a duplicate axis", {kTensorrtExecutionProvider}); //TensorRT failed
|
||||
}
|
||||
|
||||
TEST(TensorOpTest, Unsqueeze_OutOfRange) {
|
||||
|
|
@ -69,8 +69,8 @@ TEST(TensorOpTest, UnsqueezeNegAxis_3) {
|
|||
test.AddAttribute("axes", std::vector<int64_t>{-4, 1, -6});
|
||||
test.AddInput<float>("input", {2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
|
||||
test.AddOutput<float>("output", {1, 1, 1, 2, 3, 4}, std::vector<float>(2 * 3 * 4, 1.0f));
|
||||
// nGraph does not support negative axis.
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider});
|
||||
// nGraph and TensorRT does not support negative axis.
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kTensorrtExecutionProvider});
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
|
|
|||
|
|
@ -630,6 +630,7 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para
|
|||
cmd += ["-j", str(num_parallel_models)]
|
||||
|
||||
if config != 'Debug' and os.path.exists(model_dir):
|
||||
# some models in opset9 and above are not supported by TensorRT yet
|
||||
if provider == 'tensorrt':
|
||||
model_dir = os.path.join(model_dir, "opset8")
|
||||
cmd.append(model_dir)
|
||||
|
|
@ -970,9 +971,10 @@ def main():
|
|||
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data")
|
||||
|
||||
if args.use_tensorrt:
|
||||
# Disable some onnx unit tests that TensorRT parser doesn't supported yet
|
||||
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data", "simple")
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'tensorrt', False, 1)
|
||||
# Disable some onnx unit tests that TensorRT doesn't supported yet
|
||||
if not is_windows():
|
||||
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data", "simple")
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'tensorrt', False, 1)
|
||||
elif args.use_cuda:
|
||||
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', False, 2)
|
||||
elif args.x86 or platform.system() == 'Darwin':
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@ jobs:
|
|||
displayName: 'Run build script'
|
||||
inputs:
|
||||
filename: 'build.bat'
|
||||
arguments: ' --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll"
|
||||
arguments: ' --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll"
|
||||
--cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --build_dir $(buildDirectory) --config Release --use_openmp --build_wheel'
|
||||
workingFolder: "$(Build.SourcesDirectory)"
|
||||
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ jobs:
|
|||
displayName: 'Build and Test OnnxRuntime'
|
||||
inputs:
|
||||
script: |
|
||||
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_openmp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
|
||||
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_openmp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@ jobs:
|
|||
pythonInterpreter: '/usr/bin/python3'
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d tensorrt -r $(Build.BinariesDirectory) -x "--build_wheel"'
|
||||
# The latest TensorRT container (R19.09) only supports ubuntu18.04
|
||||
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d tensorrt -r $(Build.BinariesDirectory) -p 3.6 -x "--build_wheel"'
|
||||
|
||||
displayName: 'Command Line Script'
|
||||
|
||||
|
|
|
|||
|
|
@ -13,14 +13,14 @@ jobs:
|
|||
parameters:
|
||||
AgentPool : $(AgentPoolWin)
|
||||
JobName: 'Windows_CI_GPU_Dev'
|
||||
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --cuda_version 10.0'
|
||||
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --cuda_version 10.0'
|
||||
DoDebugBuild: 'false'
|
||||
DoNugetPack : 'true'
|
||||
DoCompliance: 'false'
|
||||
DoEsrp: ${{ parameters.DoEsrp }}
|
||||
BuildArch: 'amd64'
|
||||
SetVcvars: 'true'
|
||||
MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt515dll\'
|
||||
MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt6015dll\'
|
||||
EnvSetupScript: 'setup_env_cuda.bat'
|
||||
CudaVersion: '10.0'
|
||||
NuPackScript: |
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@ jobs:
|
|||
AgentPool : 'Win-GPU-CUDA10'
|
||||
DoDebugBuild: 'true'
|
||||
DoCompliance: 'false'
|
||||
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc'
|
||||
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc'
|
||||
JobName: 'Windows_CI_GPU_Dev'
|
||||
DoNugetPack: 'false'
|
||||
NuPackScript : ''
|
||||
DoTestCoverage: 'false'
|
||||
BuildArch: 'amd64'
|
||||
SetVcvars: 'true'
|
||||
MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt515dll\'
|
||||
MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt6015dll\'
|
||||
EnvSetupScript: 'setup_env_cuda.bat'
|
||||
CudaVersion: '10.0'
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ jobs:
|
|||
displayName: 'Download test data and generate cmake config'
|
||||
inputs:
|
||||
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --build_csharp --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.1.5.0" --update --cuda_version=10.0'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --build_csharp --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --update --cuda_version=10.0'
|
||||
workingDirectory: "$(Build.BinariesDirectory)"
|
||||
|
||||
- task: VSBuild@1
|
||||
|
|
@ -35,7 +35,7 @@ jobs:
|
|||
solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln'
|
||||
platform: 'x64'
|
||||
configuration: 'Debug'
|
||||
msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt515dll\'
|
||||
msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt6015dll\'
|
||||
msbuildArchitecture: 'x64'
|
||||
logProjectEvents: true
|
||||
workingFolder: '$(Build.BinariesDirectory)\Debug'
|
||||
|
|
@ -43,7 +43,7 @@ jobs:
|
|||
displayName: 'Test Debug'
|
||||
inputs:
|
||||
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.1.5.0" --test'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --test'
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: MSBuild@1
|
||||
|
|
@ -72,7 +72,7 @@ jobs:
|
|||
solution: '$(Build.BinariesDirectory)\Release\onnxruntime.sln'
|
||||
platform: 'x64'
|
||||
configuration: 'Release'
|
||||
msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt515dll\'
|
||||
msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10_trt6015dll\'
|
||||
msbuildArchitecture: 'x64'
|
||||
logProjectEvents: true
|
||||
workingFolder: '$(Build.BinariesDirectory)\Release'
|
||||
|
|
@ -81,7 +81,7 @@ jobs:
|
|||
displayName: 'Test Release'
|
||||
inputs:
|
||||
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt515dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.1.5.0" --test'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --test'
|
||||
workingFolder: "$(Build.BinariesDirectory)"
|
||||
|
||||
- task: MSBuild@1
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
# Tag: nvcr.io/nvidia/tensorrt:19.06-py3
|
||||
# Label: com.nvidia.cuda.version: 10.1.168
|
||||
# Label: com.nvidia.cudnn.version: 7.6.0
|
||||
# Ubuntu 16.04
|
||||
FROM nvcr.io/nvidia/tensorrt:19.06-py3
|
||||
# Tag: nvcr.io/nvidia/tensorrt:19.09-py3
|
||||
# Label: com.nvidia.cuda.version: 10.1.243
|
||||
# Label: com.nvidia.cudnn.version: 7.6.3
|
||||
# Ubuntu 18.04
|
||||
FROM nvcr.io/nvidia/tensorrt:19.09-py3
|
||||
|
||||
ARG PYTHON_VERSION=3.5
|
||||
ARG PYTHON_VERSION=3.6
|
||||
ARG OS_VERSION=18.04
|
||||
|
||||
ADD scripts /tmp/scripts
|
||||
RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts \
|
||||
RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION -o ${OS_VERSION} && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts \
|
||||
&& rm /usr/local/bin/cmake && rm /usr/local/bin/ctest && rm -r /usr/local/share/cmake-3.12
|
||||
|
||||
WORKDIR /root
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
while getopts p:d: parameter_Option
|
||||
while getopts p:d:o: parameter_Option
|
||||
do case "${parameter_Option}"
|
||||
in
|
||||
p) PYTHON_VER=${OPTARG};;
|
||||
d) DEVICE_TYPE=${OPTARG};;
|
||||
o) OS_VERSION=${OPTARG};;
|
||||
esac
|
||||
done
|
||||
|
||||
PYTHON_VER=${PYTHON_VER:=3.5}
|
||||
# Some Edge devices only have limited disk space, use this option to exclude some package
|
||||
DEVICE_TYPE=${DEVICE_TYPE:=Normal}
|
||||
OS_VERSION=${OS_VERSION:=16.04}
|
||||
DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
SYS_LONG_BIT=$(getconf LONG_BIT)
|
||||
|
|
@ -18,7 +20,8 @@ SYS_LONG_BIT=$(getconf LONG_BIT)
|
|||
apt-get update && apt-get install -y software-properties-common
|
||||
add-apt-repository ppa:deadsnakes/ppa
|
||||
|
||||
PACKAGE_LIST="autotools-dev \
|
||||
if [ $OS_VERSION = "16.04" ]; then
|
||||
PACKAGE_LIST="autotools-dev \
|
||||
automake \
|
||||
build-essential \
|
||||
git apt-transport-https apt-utils \
|
||||
|
|
@ -49,6 +52,41 @@ PACKAGE_LIST="autotools-dev \
|
|||
rsync libunwind8 libpng16-dev libexpat1-dev \
|
||||
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
|
||||
libprotobuf-dev libprotobuf9v5 protobuf-compiler"
|
||||
else # ubuntu18.04
|
||||
add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main"
|
||||
PACKAGE_LIST="autotools-dev \
|
||||
automake \
|
||||
build-essential \
|
||||
git apt-transport-https apt-utils \
|
||||
ca-certificates \
|
||||
pkg-config \
|
||||
wget \
|
||||
zlib1g \
|
||||
zlib1g-dev \
|
||||
libssl-dev \
|
||||
curl libcurl4-openssl-dev \
|
||||
autoconf \
|
||||
sudo \
|
||||
gfortran \
|
||||
python3-dev \
|
||||
language-pack-en \
|
||||
libopenblas-dev \
|
||||
liblttng-ust0 \
|
||||
libcurl4 \
|
||||
libssl1.0.0 \
|
||||
libkrb5-3 \
|
||||
libicu55 \
|
||||
libtinfo-dev \
|
||||
libtool \
|
||||
aria2 \
|
||||
bzip2 \
|
||||
unzip \
|
||||
zip \
|
||||
rsync libunwind8 libpng-dev libexpat1-dev \
|
||||
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
|
||||
libprotobuf-dev libprotobuf10 protobuf-compiler"
|
||||
fi
|
||||
|
||||
if [ $DEVICE_TYPE = "Normal" ]; then
|
||||
PACKAGE_LIST="$PACKAGE_LIST libedit-dev libxml2-dev python3-packaging"
|
||||
fi
|
||||
|
|
@ -68,21 +106,42 @@ if [ $SYS_LONG_BIT = "64" ]; then
|
|||
rm -rf /tmp/dotnet
|
||||
fi
|
||||
|
||||
if [ $PYTHON_VER != "3.5" ]; then
|
||||
apt-get install -y --no-install-recommends \
|
||||
python${PYTHON_VER} \
|
||||
python${PYTHON_VER}-dev
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 2
|
||||
update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
|
||||
#TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be
|
||||
#put at /usr/local/. Then there will be two pips.
|
||||
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
|
||||
fi
|
||||
if [ $OS_VERSION = "16.04" ]; then
|
||||
if [ $PYTHON_VER != "3.5" ]; then
|
||||
apt-get install -y --no-install-recommends \
|
||||
python${PYTHON_VER} \
|
||||
python${PYTHON_VER}-dev
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 2
|
||||
update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
|
||||
#TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be
|
||||
#put at /usr/local/. Then there will be two pips.
|
||||
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
|
||||
fi
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
if [ $DEVICE_TYPE = "Normal" ]; then
|
||||
aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
|
||||
tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr
|
||||
fi
|
||||
else # ubuntu18.04
|
||||
if [ $PYTHON_VER != "3.6" ]; then
|
||||
apt-get install -y --no-install-recommends \
|
||||
python${PYTHON_VER} \
|
||||
python${PYTHON_VER}-dev
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1
|
||||
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 2
|
||||
update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
|
||||
#TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be
|
||||
#put at /usr/local/. Then there will be two pips.
|
||||
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
|
||||
fi
|
||||
|
||||
if [ $DEVICE_TYPE = "Normal" ]; then
|
||||
aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
|
||||
tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
if [ $DEVICE_TYPE = "Normal" ]; then
|
||||
aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz
|
||||
tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ CUDA_VER=cuda10.0-cudnn7.3
|
|||
while getopts c:o:d:r:p:x:a:v: parameter_Option
|
||||
do case "${parameter_Option}"
|
||||
in
|
||||
#android, ubuntu16.04, manylinux2010
|
||||
#android, ubuntu16.04, manylinux2010, ubuntu18.04
|
||||
o) BUILD_OS=${OPTARG};;
|
||||
#cpu, gpu, tensorrt
|
||||
d) BUILD_DEVICE=${OPTARG};;
|
||||
|
|
@ -49,28 +49,34 @@ elif [ $BUILD_OS = "centos7" ]; then
|
|||
DOCKER_FILE=Dockerfile.centos
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
else
|
||||
if [ $BUILD_OS = "ubuntu16.04" ]; then
|
||||
OS_VER=16.04
|
||||
else
|
||||
OS_VER=18.04
|
||||
fi
|
||||
if [ $BUILD_DEVICE = "gpu" ]; then
|
||||
IMAGE="ubuntu16.04-$CUDA_VER"
|
||||
IMAGE="$BUILD_OS-$CUDA_VER"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_gpu
|
||||
if [ $CUDA_VER = "cuda9.1-cudnn7.1" ]; then
|
||||
DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9
|
||||
fi
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_DEVICE = "tensorrt" ]; then
|
||||
IMAGE="ubuntu16.04-cuda10.0-cudnn7.4-tensorrt5.0"
|
||||
# TensorRT container release 19.09
|
||||
IMAGE="$BUILD_OS-cuda10.1-cudnn7.6-tensorrt6.0"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_tensorrt
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=${OS_VER} --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_DEVICE = "openvino" ]; then
|
||||
IMAGE="ubuntu16.04-openvino"
|
||||
IMAGE="$BUILD_OS-openvino"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_openvino
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE .
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=${OS_VER} --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE .
|
||||
else
|
||||
IMAGE="ubuntu16.04"
|
||||
IMAGE="$BUILD_OS"
|
||||
if [ $BUILD_ARCH = "x86" ]; then
|
||||
IMAGE="$IMAGE.x86"
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 .
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=${OS_VER} --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 .
|
||||
else
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .
|
||||
docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=${OS_VER} --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in a new issue