Trt execution provider (#382)

* updated cmake files for trt * added trt execution provider * added trt basic test * removed trt_path action attribute * Add files via upload * Update build.py * Update trt_allocator.h * fixed issues found by reviewers * changed cast operator * added comment for custom kernel implementation * changed auto to auto& * changed to function compile APIs for TRT execution provider * changed to function compile APIs for TRT execution provider * added new DType DInt64 * adapted to the changes of onnxruntime_c_api * removed trt kernel (use function compile instead) * updated onnx-tensorrt submodule * set default memory type to TRT fused kernel * resolve merge conflict * fixed the issue that USE_CUDA conflicts with USE_TRT * construct graph by adding nodes in topological order * made changes for Windows * change buffers type * bypass HasImplementationOf check for TRT XP because TRT kernel is not registered * added domain to version info in rebuilt model proto * added trt to test option list * added DomainToVersionMap() to GraphViewer * removed Copy() * fixed broken code * format the code to clang format * used local reference to the frequently used values * fixed a couple of issues according to reviewers feedback * fixed a couple of issues according to reviewers feedback * added python binding for TRT and enable use_cuda when use_trt is on * fixed a redefinition issue * changed shared_ptr to unique_ptr on trt engines, and made a few changes required by reviewers * enabled trtexecution provider for unit tests * renamed trt to tensorrt * added tesorrt to python binding * update submodule onnx and onnx-tensorrt * made a couple of minor changes based on reviewer's feedback * added CUDA_CHECK * removed test code * fixed broken code after merge * updated onnx-tensorrt submodule * added post processing to align trt inputs/outputs with graph inputs/outputs * updated onnx submodule * added CUDA fallback for TensorRT and fixed TensorRT cmake issue * added ci pipeline for tensorrt and removed some redundent code from trt xp * fixed syntax issue * updated onnx-tensorrt submodule * fix trt build problem by: (#602) 1. Add additional /wd for debug build 2. Add io.h for additional targets 3. Bring back mb version of getopt * Update install_ubuntu.sh * Update linux-gpu-tensorrt-ci-pipeline.yml * Update linux-gpu-tensorrt-ci-pipeline.yml * Update run_build.sh * Update run_build.sh * Update run_build.sh * Update run_build.sh * fixed the issue that GetKernelRegistry returns nullptr * merged master to this branch * moved some data types to private * fixed tensorrt CI pipeline issue * customized test data for TensorRT pipeline * added onnx-tensorrt in json file and fixed an issue in ci script * added comments
2026-07-30 20:18:08 +00:00 · 2019-03-14 12:00:39 -07:00 · 2019-03-14 12:00:39 -07:00 · e8b0ae8923
commit e8b0ae8923
parent 37f7ed156e
42 changed files with 1339 additions and 61 deletions
--- a/cgmanifest.json
+++ b/cgmanifest.json
@ -280,6 +280,15 @@
               "DownloadUrl":"https://svnweb.freebsd.org/base/release/12.0.0/lib/libc/stdlib/getopt.c?revision=341707&view=co"
            }
         }
+      },
+      {
+         "component":{
+            "type":"git",
+            "git":{
+               "commitHash":"3ad2bd19d6f0a2475805e9aa5ae8734a0d60fbaa",
+               "repositoryUrl":"https://github.com/stevenlix/onnx-tensorrt.git"
+            }
+         }
      }
   ],
   "Version":1
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -64,7 +64,7 @@ option(onnxruntime_BUILD_SHARED_LIB "Build a shared library" OFF)
 option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable microsoft internal only code" OFF)
 option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF)
 option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF)
-option(onnxruntime_USE_TRT "Build with TensorRT support" OFF)
+option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF)
 option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)

 set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
@ -493,6 +493,12 @@ if (onnxruntime_USE_CUDA)
  endif()
 endif()

+if (onnxruntime_USE_TENSORRT)
+  if (WIN32)
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DELAYLOAD:nvinfer.dll /DELAYLOAD:nvinfer_plugin.dll")
+  endif()
+endif()
+
 if (onnxruntime_USE_TVM)
  if (WIN32 AND MSVC)
    # wd4100: identifier' : unreferenced formal parameter
--- a/cmake/external/onnx
+++ b/cmake/external/onnx
@ -1 +1 @@
-Subproject commit 1ec81bc6d49ccae23cd7801515feaadd13082903
+Subproject commit 1cca8733c692b57d3ebe3da044ef24f3af00006e
--- a/cmake/external/onnx-tensorrt
+++ b/cmake/external/onnx-tensorrt
@ -1 +1 @@
-Subproject commit 493487a7203d8a059a9b2288807cb700857fe5ca
+Subproject commit 3ad2bd19d6f0a2475805e9aa5ae8734a0d60fbaa
--- a/cmake/onnxruntime.cmake
+++ b/cmake/onnxruntime.cmake
@ -58,6 +58,7 @@ target_link_libraries(onnxruntime PRIVATE
    ${onnxruntime_libs}
    ${PROVIDERS_CUDA}
    ${PROVIDERS_MKLDNN}
+    ${PROVIDERS_TENSORRT}
    onnxruntime_optimizer
    onnxruntime_providers    
    onnxruntime_util
--- a/cmake/onnxruntime_providers.cmake
+++ b/cmake/onnxruntime_providers.cmake
@ -26,7 +26,10 @@ if(onnxruntime_USE_CUDA)
  set(PROVIDERS_CUDA onnxruntime_providers_cuda)
  list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda)
 endif()
-
+if(onnxruntime_USE_TENSORRT)
+  set(PROVIDERS_TENSORRT onnxruntime_providers_tensorrt)
+  list(APPEND ONNXRUNTIME_PROVIDER_NAMES tensorrt)
+endif()
 source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs})
 # add using ONNXRUNTIME_ROOT so they show up under the 'contrib_ops' folder in Visual Studio
 source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_contrib_ops_srcs})
@ -87,7 +90,7 @@ if (onnxruntime_USE_MKLDNN)
  )

  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_mkldnn_cc_srcs})
-  add_library(onnxruntime_providers_mkldnn ${onnxruntime_providers_mkldnn_cc_srcs})  
+  add_library(onnxruntime_providers_mkldnn ${onnxruntime_providers_mkldnn_cc_srcs})
  onnxruntime_add_include_to_target(onnxruntime_providers_mkldnn gsl onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
  add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES})
  set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime")
@ -96,6 +99,72 @@ if (onnxruntime_USE_MKLDNN)
  set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX)
 endif()

+if (onnxruntime_USE_TENSORRT)
+  add_definitions(-DUSE_TENSORRT=1)
+  add_definitions("-DONNX_ML=1")
+  add_definitions("-DONNX_NAMESPACE=onnx")
+  include_directories(${PROJECT_SOURCE_DIR}/external/protobuf)
+  set(CUDA_INCLUDE_DIRS ${onnxruntime_CUDA_HOME}/include)
+  set(TENSORRT_ROOT ${onnxruntime_TENSORRT_HOME})
+  include_directories(${ONNXRUNTIME_ROOT}/../cmake/external/onnx)
+  if (WIN32)
+    set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+    set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS})
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456")
+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4701 /wd4805")
+    endif()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -include algorithm")
+    set(PROTOBUF_LIBRARY libprotobuf)
+    set(DISABLED_WARNINGS_FOR_TRT /wd4267 /wd4244 /wd4996)
+  endif()
+  add_subdirectory(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt)
+  if (WIN32)
+    set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS})
+    set(CMAKE_CUDA_FLAGS ${OLD_CMAKE_CUDA_FLAGS})
+    unset(PROTOBUF_LIBRARY)
+    unset(OLD_CMAKE_CXX_FLAGS)
+    unset(OLD_CMAKE_CUDA_FLAGS)
+    set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199")
+    set_target_properties(nvonnxparser_runtime PROPERTIES LINK_FLAGS "/ignore:4199")
+    set_target_properties(trt_onnxify PROPERTIES LINK_FLAGS "/ignore:4199")
+    target_compile_definitions(trt_onnxify PRIVATE ONNXIFI_BUILD_LIBRARY=1)
+    target_sources(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc)
+    target_sources(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc)
+    target_include_directories(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include)
+    target_include_directories(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include)
+    target_compile_options(nvonnxparser_static PRIVATE /FIio.h)
+    target_compile_options(nvonnxparser PRIVATE /FIio.h)
+    target_compile_options(trt_onnxify PRIVATE /FIio.h)
+    target_compile_options(onnx2trt PRIVATE /FIio.h)
+    target_compile_options(getSupportedAPITest PRIVATE /FIio.h)
+  endif()
+  include_directories(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt)
+  include_directories(${TENSORRT_INCLUDE_DIR})
+  set(trt_link_libs cudnn ${CMAKE_DL_LIBS} ${TENSORRT_LIBRARY})
+  set(onnxparser_link_libs nvonnxparser_static nvonnxparser_plugin)
+
+  file(GLOB_RECURSE onnxruntime_providers_tensorrt_cc_srcs
+    "${ONNXRUNTIME_ROOT}/core/providers/tensorrt/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/tensorrt/*.cc"
+  )
+
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_tensorrt_cc_srcs})
+  add_library(onnxruntime_providers_tensorrt ${onnxruntime_providers_tensorrt_cc_srcs})
+  target_link_libraries(onnxruntime_providers_tensorrt ${onnxparser_link_libs} ${trt_link_libs})
+  onnxruntime_add_include_to_target(onnxruntime_providers_tensorrt onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
+  add_dependencies(onnxruntime_providers_tensorrt eigen ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  target_include_directories(onnxruntime_providers_tensorrt PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+  install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/tensorrt  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
+  set_target_properties(onnxruntime_providers_tensorrt PROPERTIES LINKER_LANGUAGE CXX)
+  set_target_properties(onnxruntime_providers_tensorrt PROPERTIES FOLDER "ONNXRuntime")
+  target_compile_definitions(onnxruntime_providers_tensorrt PRIVATE ONNXIFI_BUILD_LIBRARY=1)
+  target_compile_options(onnxruntime_providers_tensorrt PRIVATE ${DISABLED_WARNINGS_FOR_TRT})
+  if (WIN32)
+    target_compile_options(onnxruntime_providers_tensorrt INTERFACE /wd4996)
+  endif()
+endif()
+
 if (onnxruntime_ENABLE_MICROSOFT_INTERNAL)
  include(onnxruntime_providers_internal.cmake)
 endif()
--- a/cmake/onnxruntime_python.cmake
+++ b/cmake/onnxruntime_python.cmake
@ -58,6 +58,7 @@ set(onnxruntime_pybind11_state_libs
    ${onnxruntime_libs}
    ${PROVIDERS_CUDA}
    ${PROVIDERS_MKLDNN}
+    ${PROVIDERS_TENSORRT}
    onnxruntime_optimizer
    onnxruntime_providers
    onnxruntime_util
--- a/cmake/onnxruntime_unittests.cmake
+++ b/cmake/onnxruntime_unittests.cmake
@ -91,7 +91,7 @@ file(GLOB onnxruntime_test_ir_src

 file(GLOB onnxruntime_test_optimizer_src
  "${TEST_SRC_DIR}/optimizer/*.cc"
-  "${TEST_SRC_DIR}/optimizer/*.h"  
+  "${TEST_SRC_DIR}/optimizer/*.h"
  )

 set(onnxruntime_test_framework_src_patterns
@ -189,6 +189,7 @@ set(ONNXRUNTIME_TEST_LIBS
    ${onnxruntime_libs}
    ${PROVIDERS_CUDA}
    ${PROVIDERS_MKLDNN}
+    ${PROVIDERS_TENSORRT}
    onnxruntime_optimizer
    onnxruntime_providers
    onnxruntime_util
@ -205,6 +206,13 @@ set(onnxruntime_test_providers_libs
    ${ONNXRUNTIME_TEST_LIBS}
  )

+if(onnxruntime_USE_TENSORRT)
+  list(APPEND onnxruntime_test_framework_src_patterns  ${TEST_SRC_DIR}/providers/tensorrt/*)
+  list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_tensorrt)
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_tensorrt)
+  list(APPEND onnxruntime_test_providers_libs onnxruntime_providers_tensorrt)
+endif()
+
 if( NOT WIN32 AND (HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H))
  list(APPEND onnxruntime_test_providers_libs stdc++fs)
 endif()
--- a/include/onnxruntime/core/framework/func_api.h
+++ b/include/onnxruntime/core/framework/func_api.h
@ -5,7 +5,8 @@ namespace onnxruntime {
 enum DType {
  TFloat32 = 0,
  TInt32 = 1,
-  TDouble = 2
+  TDouble = 2,
+  TInt64 = 3
  //TODO: more types
 };

--- a/include/onnxruntime/core/graph/constants.h
+++ b/include/onnxruntime/core/graph/constants.h
@ -23,6 +23,6 @@ constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider";
 constexpr const char* kMklDnnExecutionProvider = "MKLDNNExecutionProvider";
 constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider";
 constexpr const char* kBrainSliceExecutionProvider = "BrainSliceExecutionProvider";
-constexpr const char* kTRTExecutionProvider = "TRTExecutionProvider";
+constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider";
 }  // namespace onnxruntime

--- a/include/onnxruntime/core/graph/graph_viewer.h
+++ b/include/onnxruntime/core/graph/graph_viewer.h
@ -97,6 +97,11 @@ class GraphViewer {
  */
  const NodeArg* GetNodeArg(const std::string& name) const;

+  /** Gets the map of operator domains to their opset versions. */
+  const std::unordered_map<std::string, int>& DomainToVersionMap() const noexcept {
+    return graph_->DomainToVersionMap();
+  }
+
 private:
  ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphViewer);

--- a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h
+++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h
@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/session/onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options);
+
+#ifdef __cplusplus
+}
+#endif
+
--- a/onnxruntime/core/framework/func_kernel.cc
+++ b/onnxruntime/core/framework/func_kernel.cc
@ -22,6 +22,8 @@ DType ORT_type_to_c_type(MLDataType type) {
    return DType::TDouble;
  else if (type == DataTypeImpl::GetType<int32_t>())
    return DType::TInt32;
+  else if (type == DataTypeImpl::GetType<int64_t>())
+    return DType::TInt64;
  else
    ORT_NOT_IMPLEMENTED("Unsupport MLType to c type.");
 }
--- a/onnxruntime/core/framework/graph_partitioner.cc
+++ b/onnxruntime/core/framework/graph_partitioner.cc
@ -180,6 +180,10 @@ Status GraphPartitioner::Partition(Graph& graph, bool export_dll, FuncManager& f
        //prepare the func kernel
        KernelDefBuilder builder;
        BuildFusedKernelDef(builder, *node);
+        if (node->GetExecutionProviderType() == onnxruntime::kTensorrtExecutionProvider) {
+          builder.SetDefaultInputsMemoryType(OrtMemTypeCPUInput);
+          builder.SetDefaultOutputMemoryType(OrtMemTypeCPUOutput);
+        }
        ORT_RETURN_IF_ERROR(fused_kernel_registry->Register(
            builder, [](const OpKernelInfo& info) { return new FunctionKernel(info); }));
      }
--- a/onnxruntime/core/framework/utils.cc
+++ b/onnxruntime/core/framework/utils.cc
@ -129,7 +129,7 @@ common::Status CopyOneInputAcrossDevices(const SessionState& session_state,
    }

    //no copy for TRT
-    if (required_provider_type == onnxruntime::kTRTExecutionProvider) {
+    if (required_provider_type == onnxruntime::kTensorrtExecutionProvider) {
      new_mlvalue = orig_mlvalue;
      break;
    }
--- a/onnxruntime/core/optimizer/transformer_memcpy.cc
+++ b/onnxruntime/core/optimizer/transformer_memcpy.cc
@ -59,7 +59,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr
    if (provider != onnxruntime::kCpuExecutionProvider &&
        provider != onnxruntime::kMklDnnExecutionProvider &&
        provider != onnxruntime::kNupharExecutionProvider &&
-        provider != onnxruntime::kTRTExecutionProvider) {
+        provider != onnxruntime::kTensorrtExecutionProvider) {
      TransformerMemcpyImpl copy_impl(graph, provider);
      modified = copy_impl.ModifyGraph(registry_manager_);
    }
@ -84,7 +84,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr

 Overview: The transformer transforms the input graph as follows:

-(1) For every initializer W that is referenced by both provider and non-provider nodes, 
+(1) For every initializer W that is referenced by both provider and non-provider nodes,
 we create a duplicate initializer W2 and change all provider nodes to reference this
 duplicate copy.

@ -182,7 +182,7 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg
    }
  } else {
    // TODO: copy between devices? i.e. multiple GPUs
-    if (node.GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider &&
+    if (node.GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider && node.GetExecutionProviderType() != onnxruntime::kTensorrtExecutionProvider &&
        !node.GetExecutionProviderType().empty()) {
      ORT_THROW("Execution type '", node.GetExecutionProviderType(), "' doesn't support memcpy ");
    }
--- a/onnxruntime/core/providers/tensorrt/symbols.txt
+++ b/onnxruntime/core/providers/tensorrt/symbols.txt
@ -0,0 +1 @@
+OrtCreateTensorrtExecutionProviderFactory
--- a/onnxruntime/core/providers/tensorrt/tensorrt_allocator.h
+++ b/onnxruntime/core/providers/tensorrt/tensorrt_allocator.h
@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/framework/allocator.h"
+
+namespace onnxruntime {
+constexpr const char* TRT = "Trt";
+
+class TensorrtPinnedAllocator : public CPUAllocator {
+ public:
+  virtual const OrtAllocatorInfo& Info() const override {
+    static OrtAllocatorInfo tensorrt_cpu_allocator_info(TRT,
+                                                   OrtAllocatorType::OrtDeviceAllocator, 0,
+                                                   OrtMemType::OrtMemTypeCPU);
+    return tensorrt_cpu_allocator_info;
+  }
+};
+
+/*! \brief The default allocator doesn't allocate anything. It's used here to let allocation
+           planner get allocator information.
+*/
+class TensorrtAllocator : public CPUAllocator {
+ public:
+  virtual const OrtAllocatorInfo& Info() const override {
+    static OrtAllocatorInfo tensorrt_default_allocator_info(TRT,
+                                                       OrtAllocatorType::OrtDeviceAllocator, 0,
+                                                       OrtMemType::OrtMemTypeDefault);
+    return tensorrt_default_allocator_info;
+  }
+};
+}  // namespace onnxruntime
--- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
+++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
@ -0,0 +1,447 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "tensorrt_execution_provider.h"
+#include "tensorrt_allocator.h"
+#include "core/framework/execution_provider.h"
+#include "core/framework/op_kernel.h"
+#include "core/framework/kernel_registry.h"
+#include "core/framework/compute_capability.h"
+#include "core/providers/cpu/cpu_execution_provider.h"
+#include "core/platform/env.h"
+#include "onnx/shape_inference/implementation.h"
+#include "cuda_runtime_api.h"
+#include "gsl/pointers"
+#include "core/graph/model.h"
+#include "cuda_runtime_api.h"
+
+using namespace std;
+using namespace ONNX_NAMESPACE;
+using namespace ::onnxruntime::logging;
+
+namespace onnxruntime {
+
+#define CHECK_CUDA(call)                         \
+  do {                                           \
+    cudaError_t status = call;                   \
+    if(status != cudaSuccess) {                  \
+      return -1;                                 \
+    }                                            \
+  } while(0)
+
+TensorrtExecutionProvider::TensorrtExecutionProvider()
+    : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider} {
+  DeviceAllocatorRegistrationInfo trt_device_info({OrtMemTypeCPU, [](int) {
+                                                     return std::make_unique<TensorrtPinnedAllocator>();
+                                                   },
+                                                   std::numeric_limits<size_t>::max()});
+  InsertAllocator(CreateAllocator(trt_device_info));
+  DeviceAllocatorRegistrationInfo default_device_info({OrtMemTypeDefault, [](int) {
+                                                         return std::make_unique<TensorrtAllocator>();
+                                                       },
+                                                       std::numeric_limits<size_t>::max()});
+  InsertAllocator(CreateAllocator(default_device_info));
+}
+
+TensorrtExecutionProvider::~TensorrtExecutionProvider() {}
+
+std::vector<std::unique_ptr<ComputeCapability>>
+TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
+                                         const std::vector<const KernelRegistry*>& /*kernel_registries*/) const {
+  // Construct modelproto from graph
+  onnxruntime::Model model(graph.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap());
+  onnxruntime::Graph& graph_build = model.MainGraph();
+  const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
+  for (const auto& index : node_index) {
+    const Node* node = graph.GetNode(index);
+    graph_build.AddNode(*node);
+  }
+  ORT_ENFORCE(graph_build.Resolve().IsOK());
+  ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
+  model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
+
+  // Serialize modelproto to string
+  string string_buf;
+  model_proto.SerializeToString(&string_buf);
+
+  // Get supported node list
+  SubGraphCollection_t supported_nodes_vector;
+  TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
+  auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
+  auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
+  auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
+  trt_parser->supportsModel(string_buf.data(), string_buf.size(), supported_nodes_vector);
+  model_proto.release_graph();
+
+  // Find inputs, initializers and outputs for each supported subgraph
+  std::vector<std::unique_ptr<ComputeCapability>> result;
+  std::set<int> supported_nodes_set;
+  int counter = 0;
+  for (const auto& group : supported_nodes_vector) {
+    if (!group.empty()) {
+      std::set<size_t> node_set(group.begin(), group.end()); //slx
+      //std::set<size_t> node_set;
+      //for (const auto& index : group) {
+      //  node_set.insert(node_index[index]);
+      //}
+      std::unique_ptr<IndexedSubGraph> sub_graph = std::make_unique<IndexedSubGraph>();
+      // Find inputs and outputs of the subgraph
+      std::map<const NodeArg *, int> fused_inputs, fused_outputs, fused_outputs_to_add;
+      std::set<const NodeArg*> erased;
+      int input_order = 0;
+      int output_order = 0;
+
+      for (const auto& index : group) {
+        supported_nodes_set.insert(index);
+        sub_graph->nodes.push_back(index);
+        const auto& node = graph.GetNode(index);
+        //supported_nodes_set.insert(node_index[index]);
+        //sub_graph->nodes.push_back(node_index[index]);
+        //const auto& node = graph.GetNode(node_index[index]);
+
+        for (const auto& input : node->InputDefs()) {
+          const auto& it = fused_outputs.find(input);
+
+          if (it != fused_outputs.end()) {
+            fused_outputs.erase(it);
+            erased.insert(input);
+          }
+          //only when input is neither in output list nor erased list, add the input to input list
+          else if (erased.find(input) == erased.end()) {
+            fused_inputs[input] = input_order++;
+          }
+        }
+
+        // For output searching, there is a special case:
+        // If node's OutputEdges are more than its outputs, meaning certain output is used more than once,
+        // if the output is connected to nodes that don't belong to the subgraph, the output need to be added
+        // to the output list
+        if (node->GetOutputEdgesCount() > node->OutputDefs().size()) {
+          for (auto it = node->OutputEdgesBegin(), end = node->OutputEdgesEnd(); it != end; ++it) {
+            const auto& node_index = it->GetNode().Index();
+            const auto& output = (it->GetNode()).InputDefs()[it->GetDstArgIndex()];
+
+            if (node_set.find(node_index) != node_set.end()) {
+              const auto& iter = fused_inputs.find(output);
+
+              if (iter != fused_inputs.end()) {
+                fused_inputs.erase(iter);
+                erased.insert(output);
+              } else if (erased.find(output) == erased.end()) {
+                fused_outputs[output] = output_order++;
+              }
+            } else {
+              fused_outputs_to_add[output] = output_order++;
+            }
+          }
+        } else {
+          for (const auto& output : node->OutputDefs()) {
+            const auto& it = fused_inputs.find(output);
+
+            if (it != fused_inputs.end()) {
+              fused_inputs.erase(it);
+              erased.insert(output);
+            }
+            // only when output is neither in input list nor erased list, add the output to output list
+            else if (erased.find(output) == erased.end()) {
+              fused_outputs[output] = output_order++;
+            }
+          }
+        }
+      }
+
+      fused_outputs.insert(fused_outputs_to_add.begin(), fused_outputs_to_add.end());
+
+      // Sort inputs and outputs by the order they were added
+      std::multimap<int, const NodeArg *> inputs, outputs;
+
+      for (auto it = fused_inputs.begin(), end = fused_inputs.end(); it != end; ++it) {
+        inputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
+      }
+
+      for (auto it = fused_outputs.begin(), end = fused_outputs.end(); it != end; ++it) {
+        outputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
+      }
+
+      // Assign inputs and outputs to subgraph's meta_def
+      auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>();
+      meta_def->name = "TRTKernel_" + std::to_string(counter++);
+      meta_def->domain = kMSDomain;
+
+      for (const auto& input : inputs) {
+        meta_def->inputs.push_back(input.second->Name());
+      }
+
+      for (const auto& output : outputs) {
+        meta_def->outputs.push_back(output.second->Name());
+      }
+
+      meta_def->since_version = 1;
+      sub_graph->SetMetaDef(meta_def);
+
+      result.push_back(std::make_unique<ComputeCapability>(std::move(sub_graph)));
+    }
+  }
+
+  return result;
+}
+
+std::shared_ptr<KernelRegistry> TensorrtExecutionProvider::GetKernelRegistry() const {
+  static std::shared_ptr<KernelRegistry> kernel_registry = std::make_shared<KernelRegistry>();
+  return kernel_registry;
+}
+
+common::Status TensorrtExecutionProvider::CopyTensor(const Tensor& src, Tensor& dst) const {
+  ORT_UNUSED_PARAMETER(src);
+  ORT_UNUSED_PARAMETER(dst);
+  return Status::OK();
+}
+
+common::Status TensorrtExecutionProvider::Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
+                                                  std::vector<NodeComputeInfo>& node_compute_funcs) {
+  for (const auto* fused_node : fused_nodes) {
+    std::vector<int> input_indexes;
+    std::vector<int> input_dim_sizes;
+    std::vector<int> output_indexes;
+    std::vector<int> output_dim_sizes;
+    std::vector<std::vector<int64_t>> output_shapes;
+
+    // Build map from input name to its index in input definitions
+    std::unordered_map<std::string, int> input_map;
+    const auto& input_defs = fused_node->InputDefs();
+    for (int i = 0, end = input_defs.size(); i < end; ++i) {
+      input_map[input_defs[i]->Name()] = i;
+    }
+
+    // Build map from output name to its index in output definitions
+    std::unordered_map<std::string, int> output_map;
+    const auto& output_defs = fused_node->OutputDefs();
+    for (int i = 0, end = output_defs.size(); i < end; ++i) {
+      output_map[output_defs[i]->Name()] = i;
+    }
+
+    // Reconstruct graph from fused node's function body
+    const auto* func_body = fused_node->GetFunctionBody();
+    if (!func_body) {
+      return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Function body is empty");
+    }
+    const Graph& graph_body = func_body->Body();
+    onnxruntime::Model model(graph_body.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph_body.DomainToVersionMap());
+    onnxruntime::Graph& graph = model.MainGraph();
+
+    for (const auto& graph_body_node : graph_body.Nodes()) {
+      graph.AddNode(graph_body_node);
+    }
+
+    ORT_ENFORCE(graph.Resolve().IsOK());
+
+    // Add initializer to graph
+    const auto& init_tensors = graph_body.GetAllInitializedTensors();
+    for (const auto& tensor : init_tensors) {
+      graph.AddInitializedTensor(*(tensor.second));
+    }
+
+    // Add fused node's outputs to graph's outputs if the outputs are not included yet
+    // for the case that node's output is connected to more than one EdgeEnd nodes and some of them don't belong to the graph
+    ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
+    const auto& graph_output = model_proto.graph().output();
+    std::set<string> graph_outputs_set;
+    for (int i = 0, end = graph_output.size(); i < end; ++i) {
+      graph_outputs_set.insert(graph_output[i].name());
+    }
+
+    const auto& graph_value_info = model_proto.graph().value_info();
+    std::vector<int> output_to_add;
+    std::vector<int> location;
+    int num_defs = output_defs.size();
+    for (int i = num_defs - 1; i >= 0; --i) {
+      const std::string& output_name = output_defs[i]->Name();
+      if (graph_outputs_set.find(output_name) == graph_outputs_set.end()) {
+        for (int j = 0, end = graph_value_info.size(); j < end; ++j) {
+          if (output_name == graph_value_info[j].name()) {
+            output_to_add.push_back(j);
+            location.push_back(num_defs - 1 - i);
+          }
+        }
+      }
+    }
+
+    // Add outputs and move them to the right places
+    auto* mutable_output = model_proto.mutable_graph()->mutable_output();
+    for (int i = 0, end = output_to_add.size(); i < end; ++i) {
+      *(mutable_output->Add()) = graph_value_info[output_to_add[i]];
+      int start_index = (*mutable_output).size() - 1;
+      int end_index = start_index - location[i];
+      for (int j = start_index; j > end_index; --j) {
+        mutable_output->SwapElements(j, j - 1);
+      }
+    }
+
+    // Set version
+    model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
+
+    // Create TensorRT engine
+    string string_buf;
+    model_proto.SerializeToString(&string_buf);
+
+    TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING);
+    auto trt_builder = unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
+    auto trt_network = unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetwork());
+    auto trt_parser = unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
+    trt_parser->parse(string_buf.data(), string_buf.size());
+    trt_builder->setMaxBatchSize(kMaxBatchSize);
+    trt_builder->setMaxWorkspaceSize(kMaxWorkSpaceSize);
+    auto trt_engine = unique_pointer<nvinfer1::ICudaEngine>(trt_builder->buildCudaEngine(*trt_network.get()));
+    ORT_ENFORCE(trt_engine != nullptr);
+
+    // Build TensorRT context
+    auto trt_context = unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
+    ORT_ENFORCE(trt_context != nullptr);
+
+    // Get input shape and binding index
+    int num_inputs = trt_network->getNbInputs();
+    input_indexes.resize(num_inputs);
+    input_dim_sizes.resize(num_inputs);
+    for (int i = 0; i < num_inputs; ++i) {
+      const std::string& name = trt_network->getInput(i)->getName();
+      size_t bindingIndex = trt_engine->getBindingIndex(name.c_str());
+      nvinfer1::Dims dimensions = trt_engine->getBindingDimensions(static_cast<int>(bindingIndex));
+      auto iter = input_map.find(name);
+      if (iter != input_map.end()) {
+        input_indexes[bindingIndex] = iter->second;
+      }
+      size_t dim_size = 1;
+      for (int j = 0, end = dimensions.nbDims; j < end; ++j) {
+        dim_size *= dimensions.d[j];
+      }
+      input_dim_sizes[bindingIndex] = dim_size;
+    }
+
+    // Get output shape and binding index
+    int num_outputs = trt_network->getNbOutputs();
+    output_indexes.resize(num_outputs);
+    output_dim_sizes.resize(num_outputs);
+    output_shapes.resize(num_outputs);
+    for (int i = 0; i < num_outputs; ++i) {
+      const std::string& name = trt_network->getOutput(i)->getName();
+      size_t bindingIndex = trt_engine->getBindingIndex(name.c_str());
+      nvinfer1::Dims dimensions = trt_engine->getBindingDimensions(static_cast<int>(bindingIndex));
+      bindingIndex -= num_inputs;
+      auto iter = output_map.find(name);
+      if (iter != output_map.end()) {
+        output_indexes[bindingIndex] = iter->second;
+      }
+      size_t dim_size = 1;
+      for (int j = 0, end = dimensions.nbDims; j < end; ++j) {
+        output_shapes[bindingIndex].push_back(dimensions.d[j]);
+        dim_size *= dimensions.d[j];
+      }
+      output_dim_sizes[bindingIndex] = dim_size;
+    }
+
+    ORT_ENFORCE(trt_engine->getNbBindings() == (num_inputs + num_outputs));
+
+    // Save engine, context and input/output info to map
+    parsers_.emplace(fused_node->Name(), std::move(trt_parser));
+    engines_.emplace(fused_node->Name(), std::move(trt_engine));
+    contexts_.emplace(fused_node->Name(), std::move(trt_context));
+    input_info_[fused_node->Name()].push_back(input_indexes);
+    input_info_[fused_node->Name()].push_back(input_dim_sizes);
+    output_info_[fused_node->Name()].push_back(output_indexes);
+    output_info_[fused_node->Name()].push_back(output_dim_sizes);
+    output_shapes_[fused_node->Name()] = output_shapes;
+
+    // Create function state
+    // TODO: remove default capture
+    NodeComputeInfo compute_info;
+    compute_info.create_state_func = [=](ComputeContext* context, FunctionState* state) {
+      std::unique_ptr<TensorrtFuncState> p = std::make_unique<TensorrtFuncState>();
+      *p = {context->allocate_func, context->release_func, context->allocator_handle, parsers_[context->node_name].get(), engines_[context->node_name].get(), contexts_[context->node_name].get(),
+            input_info_[context->node_name], output_info_[context->node_name], output_shapes_[context->node_name]};
+      *state = p.release();
+      return 0;
+    };
+
+    // Release function state
+    compute_info.release_state_func = [](FunctionState state) {
+      if (state)
+        delete static_cast<TensorrtFuncState*>(state);
+    };
+
+    // Create compute function
+    compute_info.compute_func = [](FunctionState state, ONNXRunTimeTensor* input_tensors, size_t num_inputs, ONNXRunTimeTensor* output_tensors, size_t num_outputs) {
+      ORT_UNUSED_PARAMETER(num_inputs);
+      ORT_UNUSED_PARAMETER(num_outputs);
+      TensorrtFuncState* trt_state = reinterpret_cast<TensorrtFuncState*>(state);
+      const std::vector<int>& input_indexes = (trt_state->input_info)[0];
+      const std::vector<int>& input_dim_sizes = (trt_state->input_info)[1];
+      const std::vector<int>& output_indexes = (trt_state->output_info)[0];
+      const std::vector<int>& output_dim_sizes = (trt_state->output_info)[1];
+      std::vector<std::vector<int64_t>> output_shapes = trt_state->output_shapes;
+      int num_binding_inputs = input_indexes.size();
+      int num_binding_outputs = output_indexes.size();
+      int total_bindings = num_binding_inputs + num_binding_outputs;
+      cudaStream_t stream;
+      CHECK_CUDA(cudaStreamCreate(&stream));
+      std::vector<void*> buffers(total_bindings);
+      int batch_size = 1;
+
+      // Get batch size and allocate cuda memory for inputs
+      for (int i = 0, end = num_binding_inputs; i < end; ++i) {
+        const auto& tensor_input = input_tensors[input_indexes[i]];
+        const auto& tensor_shape = tensor_input.shape;
+        const int input_batch_size = tensor_shape[0];
+        if (i > 0 && batch_size != input_batch_size) {
+          ORT_THROW("Input batch size is inconsistent");
+        }
+        batch_size = input_batch_size;
+
+        const float* input = static_cast<float*>(tensor_input.data);
+        CHECK_CUDA(cudaMalloc(&buffers[i], input_batch_size * input_dim_sizes[i] * sizeof(float)));
+        CHECK_CUDA(cudaMemcpy(buffers[i], input, input_batch_size * input_dim_sizes[i] * sizeof(float), cudaMemcpyHostToDevice));
+      }
+
+      // Allocate cuda memory for outputs
+      for (int i = 0, end = num_binding_outputs; i < end; ++i) {
+        CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float)));
+      }
+
+      // Run TRT inference
+      trt_state->context->enqueue(batch_size, &buffers[0], stream, nullptr);
+
+      // Copy TRT outputs to output tensors
+      for (int i = 0, end = num_binding_outputs; i < end; ++i) {
+        // Setup output tensor property
+        int output_index = output_indexes[i];
+        output_shapes[i].insert(output_shapes[i].begin(), batch_size);
+        output_tensors[output_index].dtype = input_tensors[0].dtype;
+        // TODO: shape inference
+        const auto& shape_size = output_shapes[i].size();
+        output_tensors[output_index].ndim = shape_size;
+        output_tensors[output_index].shape = new int64_t[shape_size];
+        memcpy(output_tensors[output_index].shape, &output_shapes[i][0], sizeof(int64_t) * shape_size);
+        output_tensors[output_index].data = (*(trt_state->test_allocate_func))(trt_state->allocator, 64, sizeof(double) * batch_size * output_dim_sizes[i]);
+
+        CHECK_CUDA(cudaMemcpy(output_tensors[output_index].data, buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float), cudaMemcpyDeviceToHost));
+      }
+
+      // Sync stream
+      cudaStreamSynchronize(stream);
+
+      // Free CUDA memory
+      cudaStreamDestroy(stream);
+
+      for (int i = 0, end = total_bindings; i < end; ++i) {
+        CHECK_CUDA(cudaFree(buffers[i]));
+      }
+
+      return 0;
+    };
+
+    node_compute_funcs.push_back(compute_info);
+  }
+
+  return Status::OK();
+}
+}  // namespace onnxruntime
+
--- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
+++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
@ -0,0 +1,100 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include <ctime>
+#include "core/common/logging/logging.h"
+#include "core/framework/op_kernel.h"
+#include "NvInfer.h"
+#include "NvOnnxParser.h"
+
+namespace onnxruntime {
+
+static const int kMaxBatchSize = 1;
+static const int kMaxWorkSpaceSize = 1 << 30;
+
+class TensorrtLogger : public nvinfer1::ILogger {
+    nvinfer1::ILogger::Severity verbosity_;
+public:
+    TensorrtLogger(Severity verbosity=Severity::kWARNING)
+        : verbosity_(verbosity) {}
+    void log(Severity severity, const char* msg) override {
+        if( severity <= verbosity_ ) {
+            time_t rawtime = std::time(0);
+            char buf[256];
+            strftime(&buf[0], 256,
+                     "%Y-%m-%d %H:%M:%S",
+                     std::gmtime(&rawtime));
+            const char* sevstr = (severity == Severity::kINTERNAL_ERROR ? "    BUG" :
+                                  severity == Severity::kERROR          ? "  ERROR" :
+                                  severity == Severity::kWARNING        ? "WARNING" :
+                                  severity == Severity::kINFO           ? "   INFO" :
+                                  "UNKNOWN");
+            LOGS_DEFAULT(WARNING) << "[" << buf << " " << sevstr << "] " << msg;
+        }
+    }
+};
+
+// Information needed to construct trt execution providers.
+struct TensorrtExecutionProviderInfo {
+  int device_id{0};
+};
+
+// Information to construct kernel function state.
+struct TensorrtFuncState {
+  AllocateFunc test_allocate_func = nullptr;
+  DestroyFunc test_release_func = nullptr;
+  AllocatorHandle allocator = nullptr;
+  nvonnxparser::IParser* parser = nullptr;
+  nvinfer1::ICudaEngine* engine = nullptr;
+  nvinfer1::IExecutionContext* context = nullptr;
+  std::vector<std::vector<int>> input_info;
+  std::vector<std::vector<int>> output_info;
+  std::vector<std::vector<int64_t>> output_shapes;
+};
+
+// Logical device representation.
+class TensorrtExecutionProvider : public IExecutionProvider {
+ public:
+  TensorrtExecutionProvider();
+  virtual ~TensorrtExecutionProvider();
+
+  std::vector<std::unique_ptr<ComputeCapability>>
+  GetCapability(const onnxruntime::GraphViewer& graph,
+                const std::vector<const KernelRegistry*>& /*kernel_registries*/) const override;
+
+  common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
+                         std::vector<NodeComputeInfo>& node_compute_funcs) override;
+
+  Status CopyTensor(const Tensor& src, Tensor& dst) const override;
+
+  const void* GetExecutionHandle() const noexcept override {
+    return nullptr;
+  }
+
+  std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
+
+ private:
+  struct InferDeleter {
+    template <typename T>
+    void operator()(T* obj) const {
+      if (obj) {
+        obj->destroy();
+      }
+    }
+  };
+
+  template <typename T>
+  using unique_pointer = std::unique_ptr<T, InferDeleter>;
+
+  int device_id_;
+  std::unordered_map<std::string, unique_pointer<nvonnxparser::IParser>> parsers_;
+  std::unordered_map<std::string, unique_pointer<nvinfer1::ICudaEngine>> engines_;
+  std::unordered_map<std::string, unique_pointer<nvinfer1::IExecutionContext>> contexts_;
+  std::unordered_map<std::string, std::vector<std::vector<int>>> input_info_;
+  std::unordered_map<std::string, std::vector<std::vector<int>>> output_info_;
+  std::unordered_map<std::string, std::vector<std::vector<int64_t>>> output_shapes_;
+};
+
+}  // namespace onnxruntime
+
--- a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc
+++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc
@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/providers/tensorrt/tensorrt_provider_factory.h"
+#include <atomic>
+#include "tensorrt_execution_provider.h"
+#include "core/session/abi_session_options_impl.h"
+
+using namespace onnxruntime;
+
+namespace onnxruntime {
+
+struct TensorrtProviderFactory : IExecutionProviderFactory {
+  TensorrtProviderFactory() {}
+  ~TensorrtProviderFactory() override {}
+
+  std::unique_ptr<IExecutionProvider> CreateProvider() override;
+};
+
+std::unique_ptr<IExecutionProvider> TensorrtProviderFactory::CreateProvider() {
+  return std::make_unique<TensorrtExecutionProvider>();
+}
+
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt() {
+  return std::make_shared<onnxruntime::TensorrtProviderFactory>();
+}
+}  // namespace onnxruntime
+
+ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options) {
+  options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_Tensorrt());
+  return nullptr;
+}
+
--- a/onnxruntime/python/onnxruntime_pybind_state.cc
+++ b/onnxruntime/python/onnxruntime_pybind_state.cc
@ -49,6 +49,9 @@
 #ifdef USE_CUDA
 #include "core/providers/cuda/cuda_provider_factory.h"
 #endif
+#ifdef USE_TENSORRT
+#include "core/providers/tensorrt/tensorrt_provider_factory.h"
+#endif
 #ifdef USE_MKLDNN
 #include "core/providers/mkldnn/mkldnn_provider_factory.h"
 #endif
@ -59,6 +62,7 @@
 namespace onnxruntime {
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(int device_id);
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt();
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Mkldnn(int use_arena);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(int device_id, const char*);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_BrainSlice(int ip, int, int, bool, const char*, const char*, const char*);
@ -193,6 +197,12 @@ inline void RegisterExecutionProvider(InferenceSession* sess, onnxruntime::IExec
 void InitializeSession(InferenceSession* sess) {
  onnxruntime::common::Status status;

+#ifdef USE_TENSORRT
+  {
+    RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_Tensorrt());
+  }
+#endif
+
 #ifdef USE_CUDA
  {
    RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(0));
--- a/onnxruntime/test/framework/test_utils.cc
+++ b/onnxruntime/test/framework/test_utils.cc
@ -18,9 +18,9 @@ IExecutionProvider* TestCudaExecutionProvider() {
 }
 #endif

-#ifdef USE_TRT
-IExecutionProvider* TestTRTExecutionProvider() {
-  static TRTExecutionProvider trt_provider;
+#ifdef USE_TENSORRT
+IExecutionProvider* TestTensorrtExecutionProvider() {
+  static TensorrtExecutionProvider trt_provider;
  return &trt_provider;
 }
 #endif
--- a/onnxruntime/test/framework/test_utils.h
+++ b/onnxruntime/test/framework/test_utils.h
@ -9,8 +9,8 @@
 #ifdef USE_CUDA
 #include "core/providers/cuda/cuda_execution_provider.h"
 #endif
-#ifdef USE_TRT 
-#include "core/providers/trt/trt_execution_provider.h"
+#ifdef USE_TENSORRT 
+#include "core/providers/tensorrt/tensorrt_execution_provider.h"
 #endif

 namespace onnxruntime {
@ -21,8 +21,8 @@ IExecutionProvider* TestCPUExecutionProvider();
 IExecutionProvider* TestCudaExecutionProvider();
 #endif

-#ifdef USE_TRT
-IExecutionProvider* TestTRTExecutionProvider();
+#ifdef USE_TENSORRT
+IExecutionProvider* TestTensorrtExecutionProvider();
 #endif

 template <typename T>
--- a/onnxruntime/test/onnx/main.cc
+++ b/onnxruntime/test/onnx/main.cc
@ -32,7 +32,7 @@ void usage() {
      "\t-r [repeat]: Specifies the number of times to repeat\n"
      "\t-v: verbose\n"
      "\t-n [test_case_name]: Specifies a single test case to run.\n"
-      "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda' or 'mkldnn'. Default: 'cpu'.\n"
+      "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn' or 'tensorrt'. Default: 'cpu'.\n"
      "\t-x: Use parallel executor, default (without -x): sequential executor.\n"
      "\t-h: help\n");
 }
@ -81,7 +81,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
  bool enable_cuda = false;
  bool enable_mkl = false;
  bool enable_nuphar = false;
-  bool enable_trt = false;
+  bool enable_tensorrt = false;
  OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING;
  {
    int ch;
@ -131,8 +131,8 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
            enable_mkl = true;
          } else if (!CompareCString(optarg, ORT_TSTR("nuphar"))) {
            enable_nuphar = true;
-          } else if (!CompareCString(optarg, ORT_TSTR("trt"))) {
-            enable_trt = true;
+          } else if (!CompareCString(optarg, ORT_TSTR("tensorrt"))) {
+            enable_tensorrt = true;
          } else {
            usage();
            return -1;
@ -188,6 +188,15 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
      sf.EnableSequentialExecution();
    else
      sf.DisableSequentialExecution();
+    if (enable_tensorrt) {
+#ifdef USE_TENSORRT
+      ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf));
+      ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
+#else
+      fprintf(stderr, "TensorRT is not supported in this build");
+      return -1;
+#endif
+    }
    if (enable_cuda) {
 #ifdef USE_CUDA
      ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
@ -210,14 +219,6 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
 #else
      fprintf(stderr, "MKL-DNN is not supported in this build");
      return -1;
-#endif
-    }
-    if (enable_trt) {
-#ifdef USE_TRT
-      ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_TRT(sf));
-#else
-      fprintf(stderr, "TensorRT is not supported in this build");
-      return -1;
 #endif
    }
    TestEnv args(tests, stat, sf);
@ -249,7 +250,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) {
      {"BatchNorm3d_eval", "disable reason"},
      {"BatchNorm3d_momentum_eval", "disable reason"},
      {"constantofshape_float_ones", "test data bug"},
-      {"constantofshape_int_zeros", "test data bug"},      
+      {"constantofshape_int_zeros", "test data bug"},
      {"GLU", "disable reason"},
      {"GLU_dim", "disable reason"},
      {"Linear", "disable reason"},
--- a/onnxruntime/test/perftest/command_args_parser.cc
+++ b/onnxruntime/test/perftest/command_args_parser.cc
@ -28,7 +28,7 @@ namespace perftest {
      "Options:\n"
      "\t-m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'.\n"
      "\t\tProvide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. Default:'duration'.\n"
-      "\t-e [cpu|cuda|mkldnn]: Specifies the provider 'cpu','cuda','mkldnn'. Default:'cpu'.\n"
+      "\t-e [cpu|cuda|mkldnn|tensorrt]: Specifies the provider 'cpu','cuda','mkldnn' or 'tensorrt'. Default:'cpu'.\n"
      "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n"
      "\t-t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600.\n"
      "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n"
@ -63,8 +63,8 @@ namespace perftest {
          test_config.machine_config.provider_type_name = onnxruntime::kMklDnnExecutionProvider;
        } else if (!CompareCString(optarg, ORT_TSTR("brainslice"))) {
          test_config.machine_config.provider_type_name = onnxruntime::kBrainSliceExecutionProvider;
-        } else if (!CompareCString(optarg, ORT_TSTR("trt"))) {
-          test_config.machine_config.provider_type_name = onnxruntime::kTRTExecutionProvider;
+        } else if (!CompareCString(optarg, ORT_TSTR("tensorrt"))) {
+          test_config.machine_config.provider_type_name = onnxruntime::kTensorrtExecutionProvider;
        } else {
          return false;
        }
--- a/onnxruntime/test/perftest/performance_runner.cc
+++ b/onnxruntime/test/perftest/performance_runner.cc
@ -122,9 +122,10 @@ bool PerformanceRunner::Initialize() {
    fprintf(stderr, "Nuphar is not supported in this build");
    return false;
 #endif
-  } else if (provider_name == onnxruntime::kTRTExecutionProvider) {
-#ifdef USE_TRT
-    ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_TRT(sf));
+  } else if (provider_name == onnxruntime::kTensorrtExecutionProvider) {
+#ifdef USE_TENSORRT
+    ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf));
+    ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0));
 #else
    fprintf(stderr, "TensorRT is not supported in this build");
    return false;
--- a/onnxruntime/test/providers/provider_test_utils.cc
+++ b/onnxruntime/test/providers/provider_test_utils.cc
@ -408,6 +408,7 @@ void OpTester::Run(ExpectResult expect_result,
        kMklDnnExecutionProvider,
        kNupharExecutionProvider,
        kBrainSliceExecutionProvider,
+        kTensorrtExecutionProvider,
    };

    bool has_run = false;
@ -446,6 +447,8 @@ void OpTester::Run(ExpectResult expect_result,
          execution_provider = DefaultNupharExecutionProvider();
        else if (provider_type == onnxruntime::kBrainSliceExecutionProvider)
          execution_provider = DefaultBrainSliceExecutionProvider();
+        else if (provider_type == onnxruntime::kTensorrtExecutionProvider)
+          execution_provider = DefaultTensorrtExecutionProvider();
        // skip if execution provider is disabled
        if (execution_provider == nullptr)
          continue;
--- a/onnxruntime/test/providers/trt/trt_basic_test.cc
+++ b/onnxruntime/test/providers/trt/trt_basic_test.cc
@ -0,0 +1,103 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/session/inference_session.h"
+#include "test/providers/provider_test_utils.h"
+#include "test/framework/test_utils.h"
+#include "gtest/gtest.h"
+#include "core/providers/trt/trt_execution_provider.h"
+
+using namespace std;
+using namespace ONNX_NAMESPACE;
+using namespace ::onnxruntime::logging;
+
+namespace onnxruntime {
+
+namespace test {
+void VerifyOutputs(const std::vector<MLValue>& fetches,
+                   const std::vector<int64_t>& expected_dims,
+                   const std::vector<float>& expected_values) {
+  ASSERT_EQ(1, fetches.size());
+  auto& rtensor = fetches.front().Get<Tensor>();
+  TensorShape expected_shape(expected_dims);
+  ASSERT_EQ(expected_shape, rtensor.Shape());
+  const std::vector<float> found(rtensor.template Data<float>(), rtensor.template Data<float>() + expected_values.size());
+  ASSERT_EQ(expected_values, found);
+}
+
+TEST(TensorrtExecutionProviderTest, FunctionTest) {
+  onnxruntime::Model model("graph_1");
+  auto& graph = model.MainGraph();
+  std::vector<onnxruntime::NodeArg*> inputs;
+  std::vector<onnxruntime::NodeArg*> outputs;
+
+  // FLOAT tensor.
+  ONNX_NAMESPACE::TypeProto float_tensor;
+  float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
+
+  auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor);
+  auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor);
+  inputs.push_back(&input_arg_1);
+  inputs.push_back(&input_arg_2);
+  auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor);
+  outputs.push_back(&output_arg);
+  graph.AddNode("node_1", "Add", "node 1.", inputs, outputs);
+
+  auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor);
+  inputs.clear();
+  inputs.push_back(&output_arg);
+  inputs.push_back(&input_arg_3);
+  auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor);
+  outputs.clear();
+  outputs.push_back(&output_arg_2);
+  graph.AddNode("node_2", "Add", "node 2.", inputs, outputs);
+
+  auto status = graph.Resolve();
+  ASSERT_TRUE(status.IsOK());
+  std::string model_file_name = "trt_execution_provider_test_graph.onnx";
+  status = onnxruntime::Model::Save(model, model_file_name);
+
+  std::vector<int64_t> dims_mul_x = {1, 3, 2};
+  std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+  MLValue ml_value_x;
+  CreateMLValue<float>(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_x);
+  MLValue ml_value_y;
+  CreateMLValue<float>(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_y);
+  MLValue ml_value_z;
+  CreateMLValue<float>(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_z);
+  NameMLValMap feeds;
+  feeds.insert(std::make_pair("X", ml_value_x));
+  feeds.insert(std::make_pair("Y", ml_value_y));
+  feeds.insert(std::make_pair("Z", ml_value_z));
+
+  // prepare outputs
+  std::vector<std::string> output_names;
+  output_names.push_back("M");
+  std::vector<MLValue> fetches;
+
+  // prepare expected inputs and outputs
+  std::vector<int64_t> expected_dims_mul_m = {1, 3, 2};
+  std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
+
+  SessionOptions so;
+  so.session_logid = "TensorrtExecutionProviderTest.FunctionTest";
+  RunOptions run_options;
+  run_options.run_tag = so.session_logid;
+
+  InferenceSession session_object{so};
+  session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::TensorrtExecutionProvider>());
+  status = session_object.Load(model_file_name);
+  ASSERT_TRUE(status.IsOK());
+  status = session_object.Initialize();
+  ASSERT_TRUE(status.IsOK());
+
+  // Now run
+  status = session_object.Run(run_options, feeds, output_names, &fetches);
+  ASSERT_TRUE(status.IsOK());
+  VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
+}
+}  // namespace test
+}  // namespace onnxruntime
--- a/onnxruntime/test/util/default_providers.cc
+++ b/onnxruntime/test/util/default_providers.cc
@ -13,7 +13,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(i
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Mkldnn(int use_arena);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(int device_id, const char*);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_BrainSlice(uint32_t ip, int, int, bool, const char*, const char*, const char*);
-std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_TRT();
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt();

 namespace test {

@ -21,6 +21,14 @@ std::unique_ptr<IExecutionProvider> DefaultCpuExecutionProvider(bool enable_aren
  return CreateExecutionProviderFactory_CPU(enable_arena)->CreateProvider();
 }

+std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider() {
+#ifdef USE_TENSORRT
+  return CreateExecutionProviderFactory_Tensorrt()->CreateProvider();
+#else
+  return nullptr;
+#endif
+}
+
 std::unique_ptr<IExecutionProvider> DefaultCudaExecutionProvider() {
 #ifdef USE_CUDA
  return CreateExecutionProviderFactory_CUDA(0)->CreateProvider();
@ -54,13 +62,5 @@ std::unique_ptr<IExecutionProvider> DefaultBrainSliceExecutionProvider() {
 #endif
 }

-std::unique_ptr<IExecutionProvider> DefaultTRTExecutionProvider() {
-#ifdef USE_TRT
-  return CreateExecutionProviderFactory_TRT()->CreateProvider();
-#else
-  return nullptr;
-#endif
-}
-
 }  // namespace test
 }  // namespace onnxruntime
--- a/onnxruntime/test/util/include/default_providers.h
+++ b/onnxruntime/test/util/include/default_providers.h
@ -12,7 +12,7 @@ std::unique_ptr<IExecutionProvider> DefaultCudaExecutionProvider();
 std::unique_ptr<IExecutionProvider> DefaultMkldnnExecutionProvider(bool enable_arena = true);
 std::unique_ptr<IExecutionProvider> DefaultNupharExecutionProvider();
 std::unique_ptr<IExecutionProvider> DefaultBrainSliceExecutionProvider();
-std::unique_ptr<IExecutionProvider> DefaultTRTExecutionProvider();
+std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider();

 }  // namespace test
 }  // namespace onnxruntime
--- a/onnxruntime/test/util/include/providers.h
+++ b/onnxruntime/test/util/include/providers.h
@ -16,6 +16,6 @@
 #if USE_BRAINSLICE
 #include "core/providers/brainslice/brainslice_provider_factory.h"
 #endif
-#if USE_TRT
-#include "core/providers/trt/trt_provider_factory.h"
+#ifdef USE_TENSORRT
+#include "core/providers/tensorrt/tensorrt_provider_factory.h"
 #endif
--- a/onnxruntime/test/win_getopt/mb/getopt.cc
+++ b/onnxruntime/test/win_getopt/mb/getopt.cc
@ -0,0 +1,117 @@
+/*	$NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $	*/
+
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1987, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int opterr = 1, /* if error message should be printed */
+    optind = 1, /* index into parent argv vector */
+    optopt,     /* character checked for validity */
+    optreset;   /* reset getopt */
+char* optarg;   /* argument associated with option */
+
+#define BADCH (int)'?'
+#define BADARG (int)':'
+#define EMSG ""
+
+/*
+ * getopt --
+ *	Parse argc/argv argument vector.
+ */
+int getopt(int nargc, char* const nargv[], const char* ostr) {
+  static char* place = EMSG; /* option letter processing */
+  char* oli;                 /* option letter list index */
+
+  if (optreset || *place == 0) { /* update scanning pointer */
+    optreset = 0;
+    place = nargv[optind];
+    if (optind >= nargc || *place++ != '-') {
+      /* Argument is absent or is not an option */
+      place = EMSG;
+      return (-1);
+    }
+    optopt = *place++;
+    if (optopt == '-' && *place == 0) {
+      /* "--" => end of options */
+      ++optind;
+      place = EMSG;
+      return (-1);
+    }
+    if (optopt == 0) {
+      /* Solitary '-', treat as a '-' option
+                           if the program (eg su) is looking for it. */
+      place = EMSG;
+      if (strchr(ostr, '-') == NULL) return (-1);
+      optopt = '-';
+    }
+  } else
+    optopt = *place++;
+
+  /* See if option letter is one the caller wanted... */
+  if (optopt == ':' || (oli = (char*)strchr(ostr, optopt)) == NULL) {
+    if (*place == 0) ++optind;
+    if (opterr && *ostr != ':') (void)fprintf(stderr, "illegal option -- %c\n", optopt);
+    return (BADCH);
+  }
+
+  /* Does this option need an argument? */
+  if (oli[1] != ':') {
+    /* don't need argument */
+    optarg = NULL;
+    if (*place == 0) ++optind;
+  } else {
+    /* Option-argument is either the rest of this argument or the
+                   entire next argument. */
+    if (*place)
+      optarg = place;
+    else if (oli[2] == ':')
+      /*
+       * GNU Extension, for optional arguments if the rest of
+       * the argument is empty, we return NULL
+       */
+      optarg = NULL;
+    else if (nargc > ++optind)
+      optarg = nargv[optind];
+    else {
+      /* option-argument absent */
+      place = EMSG;
+      if (*ostr == ':') return (BADARG);
+      if (opterr) (void)fprintf(stderr, "option requires an argument -- %c\n", optopt);
+      return (BADCH);
+    }
+    place = EMSG;
+    ++optind;
+  }
+  return (optopt); /* return option letter */
+}
--- a/onnxruntime/test/win_getopt/mb/include/getopt.h
+++ b/onnxruntime/test/win_getopt/mb/include/getopt.h
@ -0,0 +1,42 @@
+/*	$NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $	*/
+
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1987, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#pragma once
+
+extern int opterr,   /* if error message should be printed */
+    optind,          /* index into parent argv vector */
+    optopt,          /* character checked for validity */
+    optreset;        /* reset getopt */
+extern char* optarg; /* argument associated with option */
+
+int getopt(int nargc, char* const nargv[], const char* ostr);
--- a/onnxruntime/test/win_getopt/mb/include/unistd.h
+++ b/onnxruntime/test/win_getopt/mb/include/unistd.h
@ -0,0 +1,5 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "getopt.h"
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -128,8 +128,8 @@ Use the individual flags to only run the specified stages.
    parser.add_argument("--brain_slice_package_name", help="Name of brain slice packages")
    parser.add_argument("--brain_slice_client_package_name", help="Name of brainslice client package")
    parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar")
-    parser.add_argument("--use_trt", action='store_true', help="Build with trt")
-    parser.add_argument("--trt_path", action='store_true', help="Path to trt dir")
+    parser.add_argument("--use_tensorrt", action='store_true', help="Build with TensorRT")
+    parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir")
    return parser.parse_args()

 def resolve_executable_path(command_or_path):
@ -281,7 +281,7 @@ def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure
                log.debug("creating shortcut %s -> %s"  % (src_model_dir, dest_model_dir))
                run_subprocess(['mklink', '/D', '/J', dest_model_dir, src_model_dir], shell=True)

-def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
+def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, tensorrt_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
    log.info("Generating CMake build tree")
    cmake_dir = os.path.join(source_dir, "cmake")
    # TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind)
@ -294,6 +294,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                 "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"),
                 "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() or not args.use_nsync else "ON"),
                 "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""),
+                 "-Donnxruntime_CUDA_HOME=" + (cuda_home if args.use_cuda else ""),
                 "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"),
                 "-Donnxruntime_ENABLE_PYTHON=" + ("ON" if args.enable_pybind else "OFF"),
                 "-Donnxruntime_BUILD_CSHARP=" + ("ON" if args.build_csharp else "OFF"),
@ -309,7 +310,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                 "-Donnxruntime_USE_BRAINSLICE=" + ("ON" if args.use_brainslice else "OFF"),
                 "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"),
                 "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), 
-                 "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"),
+                 "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"),
+                 "-Donnxruntime_TENSORRT_HOME=" + (tensorrt_home if args.use_tensorrt else ""),
                  # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script)
                 "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"),
                 "-Donnxruntime_BUILD_x86=" + ("ON" if args.x86 else "OFF"),
@ -322,9 +324,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
            "-Donnxruntime_BS_CLIENT_PACKAGE=%s/%s" % (args.brain_slice_package_path, args.brain_slice_client_package_name),
            "-Donnxruntime_BRAINSLICE_dynamic_lib_PATH=%s/%s" % (args.brain_slice_package_path, bs_shared_lib_name)]

-    if args.use_trt:
-        cmake_args += ["-DTENSORRT_ROOT=%s" % args.trt_path]
-
    if args.use_llvm:
        cmake_args += ["-DLLVM_DIR=%s" % args.llvm_path]

@ -457,6 +456,22 @@ def setup_cuda_vars(args):

    return cuda_home, cudnn_home

+def setup_tensorrt_vars(args):
+
+    tensorrt_home = ""
+
+    if (args.use_tensorrt):
+        tensorrt_home = args.tensorrt_home if args.tensorrt_home else os.getenv("TENSORRT_HOME")
+
+        tensorrt_home_valid = (tensorrt_home != None and os.path.exists(tensorrt_home))
+
+        if (not tensorrt_home_valid):
+            raise BuildError("tensorrt_home paths must be specified and valid.",
+                             "tensorrt_home='{}' valid={}."
+                             .format(tensorrt_home, tensorrt_home_valid))
+
+    return tensorrt_home
+
 def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_python_tests, enable_tvm = False):
    for config in configs:
        log.info("Running tests for %s configuration", config)
@ -509,6 +524,8 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para
          cmd += ["-j", str(num_parallel_models)]

        if config != 'Debug' and os.path.exists(model_dir):
+          if provider == 'tensorrt':
+            model_dir = os.path.join(model_dir, "opset8")
          cmd.append(model_dir)
        if os.path.exists(onnx_test_data_dir):
          cmd.append(onnx_test_data_dir)
@ -521,7 +538,10 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para
          else:
            run_subprocess([exe,'-x'] + cmd, cwd=cwd)
        else:
-          run_subprocess([exe] + cmd, cwd=cwd)
+          if provider == 'tensorrt':
+            run_subprocess([exe, '-c', '1'] + cmd, cwd=cwd)
+          else:
+            run_subprocess([exe] + cmd, cwd=cwd)

 def build_python_wheel(source_dir, build_dir, configs, use_cuda):
    for config in configs:
@ -578,6 +598,9 @@ def main():
        else:
            args.test = True

+    if args.use_tensorrt:
+        args.use_cuda = True
+
    if args.build_wheel:
        args.enable_pybind = True
    
@ -596,6 +619,9 @@ def main():
    # if using cuda, setup cuda paths and env vars
    cuda_home, cudnn_home = setup_cuda_vars(args)

+    # if using tensorrt, setup tensorrt paths
+    tensorrt_home = setup_tensorrt_vars(args)
+
    os.makedirs(build_dir, exist_ok=True)

    log.info("Build started")
@ -649,7 +675,7 @@ def main():
        elif args.arm or args.arm64:
            path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')

-        generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines,
+        generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, tensorrt_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines,
                            args, cmake_extra_args)

    if (args.clean):
@ -666,7 +692,10 @@ def main():
            onnx_test_data_dir = '/data/onnx'
            if is_windows() or not os.path.exists(onnx_test_data_dir):
                onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data")
-            if args.use_cuda:
+            if args.use_tensorrt:
+              onnx_test_data_dir = ''
+              run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'tensorrt', False, 1)            
+            elif args.use_cuda:
              run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', False, 2)
            elif args.x86 or platform.system() == 'Darwin':
              run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, True, 1)
--- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml
@ -0,0 +1,16 @@
+jobs:
+- job: Linux_CI_GPU_TENSORRT_Dev
+  pool: Linux-GPU-CUDA10
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+
+    # There are some tests in 20190130.zip that TensorRT can't run. Instead here use 20181210 opset8 for TensorRT test.
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d tensorrt -r $(Build.BinariesDirectory) -x "--test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip --test_data_checksum a966def7447f4ff04f5665bca235b3f3"'
+
+      displayName: 'Command Line Script'
+
+    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+      displayName: 'Component Detection'
+      condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
+
+    - template: templates/clean-agent-build-directory-step.yml
--- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
@ -0,0 +1,144 @@
+jobs:
+- job: Windows_CI_GPU_Dev
+  pool: Win-GPU-CUDA10
+  variables:
+    buildDirectory: '$(Build.BinariesDirectory)'
+    CUDA_VERSION: '10.0'
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+    - task: NuGetCommand@2
+      displayName: 'NuGet restore'
+      inputs:
+        restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
+        feedsToUse: config
+        nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config'
+        restoreDirectory: '$(Build.SourcesDirectory)\csharp'
+    - task: UniversalPackages@0
+      displayName: 'Download python'
+      inputs:
+        command: download
+        vstsFeed: '$(System.TeamProject)'
+        vstsFeedPackage: 'miniconda3_win64'
+        vstsPackageVersion: '4.5.11'
+        downloadDirectory: '$(Build.BinariesDirectory)\python'
+    - task: CmdLine@1
+      displayName: 'Run python installer'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\python\installer.exe'
+        arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
+      timeoutInMinutes: 10
+    - task: BatchScript@1
+      displayName: 'setup env'
+      inputs:
+        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Install conda modules'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe'
+        arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy'
+      timeoutInMinutes: 10
+
+
+    - task: CmdLine@1
+      displayName: 'Download cmake'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
+
+    - task: BatchScript@1
+      displayName: 'Setup VS2017 env vars'
+      inputs:
+        filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat'
+        arguments: 'amd64 -vcvars_ver=14.11'
+        modifyEnvironment: true
+
+    - task: CmdLine@1
+      displayName: 'Download test data and generate cmake config'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
+        workingDirectory: "$(Build.BinariesDirectory)"
+
+    - task: VSBuild@1
+      displayName: 'Build Debug'
+      inputs:
+        solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln'
+        platform: 'x64'
+        configuration: 'Debug'
+        msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\'
+        msbuildArchitecture: 'x64'
+        logProjectEvents: true
+        workingFolder: '$(Build.BinariesDirectory)\Debug'
+    - task: BatchScript@1
+      displayName: 'Test Debug'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        workingFolder: '$(Build.BinariesDirectory)'
+    - task: VSBuild@1
+      displayName: 'Build C# Debug'
+      inputs:
+        solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
+        configuration: 'Debug'
+        platform: 'any cpu'
+        restoreNugetPackages: false
+        msbuildArchitecture: 'x64'
+        workingFolder: '$(Build.SourcesDirectory)\csharp'
+        msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)'
+
+    - task: VSTest@2
+      displayName: 'VsTest - C# Debug'
+      inputs:
+        testAssemblyVer2: '**\bin\Debug\**\*Tests.dll'
+        searchFolder: '$(Build.SourcesDirectory)\csharp\test'
+        runInParallel: true
+        configuration: Debug
+
+    - task: VSBuild@1
+      displayName: 'Build Release'
+      inputs:
+        solution: '$(Build.BinariesDirectory)\Release\onnxruntime.sln'
+        platform: 'x64'
+        configuration: 'Release'
+        msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\'
+        msbuildArchitecture: 'x64'
+        logProjectEvents: true
+        workingFolder: '$(Build.BinariesDirectory)\Release'
+
+    - task: BatchScript@1
+      displayName: 'Test Release'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --use_openmp --use_mkldnn --build_shared_lib  --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
+        workingFolder: "$(Build.BinariesDirectory)"
+
+    - task: VSBuild@1
+      displayName: 'Build c# Release'
+      inputs:
+        solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
+        platform: 'any cpu'
+        configuration: 'Release'
+        msbuildArchitecture: 'x64'
+        restoreNugetPackages: false
+        workingFolder: '$(Build.SourcesDirectory)\csharp'
+        msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)'
+
+    - task: VSTest@2
+      displayName: 'VsTest - C# Release'
+      inputs:
+        testAssemblyVer2: '**\bin\Release\**\*Tests.dll'
+        searchFolder: '$(Build.SourcesDirectory)\csharp\test'
+        runInParallel: true
+        configuration: Release
+
+    - task: PublishTestResults@2
+      displayName: 'Publish unit test results'
+      inputs:
+        testResultsFiles: '**\*.results.xml'
+        searchFolder: '$(Build.BinariesDirectory)'
+        testRunTitle: 'Unit Test Run'
+      condition: succeededOrFailed()
+    - template: templates/clean-agent-build-directory-step.yml
+
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
@ -0,0 +1,46 @@
+# Tag: nvcr.io/nvidia/tensorrt:19.02-py3
+# Label: com.nvidia.cuda.version: 10.0.130
+# Label: com.nvidia.cudnn.version: 7.4.2
+# Ubuntu 16.04
+FROM nvcr.io/nvidia/tensorrt:19.02-py3
+
+ARG PYTHON_VERSION=3.5
+
+ADD scripts /tmp/scripts
+ENV PATH="/opt/cmake/bin:${PATH}"
+RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
+
+WORKDIR /root
+
+# Allow configure to pick up GDK and CuDNN where it expects it.
+# (Note: $CUDNN_VERSION is defined by NVidia's base image)
+RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \
+    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \
+    ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \
+    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \
+    ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \
+    ln -s /usr/local/cudnn{-$_CUDNN_VERSION,}
+
+# Build and Install LLVM
+ARG LLVM_VERSION=6.0.1
+RUN cd /tmp && \
+    wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \
+    xz -d llvm-$LLVM_VERSION.src.tar.xz && \
+    tar xvf llvm-$LLVM_VERSION.src.tar && \
+    cd llvm-$LLVM_VERSION.src && \
+    mkdir -p build && \
+    cd build && \
+    cmake .. -DCMAKE_BUILD_TYPE=Release && \
+    cmake --build . -- -j$(nproc) && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \
+    cd /tmp && \
+    rm -rf llvm*
+
+ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
+
+ARG BUILD_USER=onnxruntimedev
+ARG BUILD_UID=1000
+WORKDIR /home/$BUILD_USER
+RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
+USER $BUILD_USER
+
--- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh
+++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh
@ -62,6 +62,7 @@ if [ $PYTHON_VER!="3.5" ]; then
    update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
 fi

+/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0
 rm -rf /var/lib/apt/lists/*

--- a/tools/ci_build/github/linux/run_build.sh
+++ b/tools/ci_build/github/linux/run_build.sh
@ -23,6 +23,16 @@ if [ $BUILD_DEVICE = "gpu" ]; then
        --cuda_home /usr/local/cuda \
        --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda --build_shared_lib $BUILD_EXTR_PAR
    /home/onnxruntimedev/Release/onnx_test_runner -e cuda /data/onnx
+elif [ $BUILD_DEVICE = "tensorrt" ]; then
+    _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
+    python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \
+        --config Release \
+        --enable_onnx_tests \
+        --parallel --build_shared_lib \
+        --use_tensorrt --tensorrt_home /workspace/tensorrt \
+        --use_openmp \
+        --cuda_home /usr/local/cuda \
+        --cudnn_home /usr/local/cuda --build_shared_lib $BUILD_EXTR_PAR
 else
    python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \
        --config Debug Release --build_shared_lib \
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@ -10,7 +10,7 @@ do case "${parameter_Option}"
 in
 #ubuntu16.04
 o) BUILD_OS=${OPTARG};;
-#cpu, gpu
+#cpu, gpu, tensorrt
 d) BUILD_DEVICE=${OPTARG};;
 r) BUILD_DIR=${OPTARG};;
 #python version: 3.6 3.7 (absence means default 3.5)
@ -35,6 +35,10 @@ if [ $BUILD_DEVICE = "gpu" ]; then
    DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9
    fi
    docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
+elif [ $BUILD_DEVICE = "tensorrt" ]; then
+    IMAGE="ubuntu16.04-$CUDA_VER"
+    DOCKER_FILE=Dockerfile.ubuntu_tensorrt
+    docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
 else
    IMAGE="ubuntu16.04"
    if [ $BUILD_ARCH = "x86" ]; then
@ -57,6 +61,17 @@ if [ $BUILD_DEVICE = "cpu" ]; then
        "onnxruntime-$IMAGE" \
        /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \
         -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" &
+elif [ $BUILD_DEVICE = "gpu" ]; then
+    docker rm -f "onnxruntime-$BUILD_DEVICE" || true
+    nvidia-docker run --rm -h $HOSTNAME \
+        --rm \
+        --name "onnxruntime-$BUILD_DEVICE" \
+        --volume "$SOURCE_ROOT:/onnxruntime_src" \
+        --volume "$BUILD_DIR:/home/onnxruntimedev" \
+        --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \
+        "onnxruntime-$IMAGE" \
+        /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \
+        -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" &
 else
    docker rm -f "onnxruntime-$BUILD_DEVICE" || true
    nvidia-docker run --rm -h $HOSTNAME \