diff --git a/include/onnxruntime/core/framework/tensor.h b/include/onnxruntime/core/framework/tensor.h
index ce5fc39d40..df0dc8e859 100644
--- a/include/onnxruntime/core/framework/tensor.h
+++ b/include/onnxruntime/core/framework/tensor.h
@@ -59,6 +59,7 @@ using BufferNakedPtr = void*;
 class Tensor final {
  public:
   static std::unique_ptr<Tensor> Create(MLDataType p_type, const TensorShape& shape, std::shared_ptr<IAllocator> allocator) { return std::make_unique<Tensor>(p_type, shape, allocator); }
+  static std::unique_ptr<Tensor> Create(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc, ptrdiff_t offset = 0) { return std::make_unique<Tensor>(p_type, shape, p_data, alloc, offset); }
 
   Tensor() = default;  // to allow creating vector<Tensor> to support seq(tensor)
 
diff --git a/onnxruntime/core/framework/ort_value_tensor_slicer.cc b/onnxruntime/core/framework/ort_value_tensor_slicer.cc
index 6d2a0b08e2..b916aa90ba 100644
--- a/onnxruntime/core/framework/ort_value_tensor_slicer.cc
+++ b/onnxruntime/core/framework/ort_value_tensor_slicer.cc
@@ -80,12 +80,7 @@ void OrtValueTensorSlicer<T>::Iterator::MaterializeMLValue() const {
   //
   // TODO: Ideally we could avoid the overhead of creating a new Tensor (mainly cost of copying type and shape info)
   // and would simply update Tensor::p_data_ given all other info remains constant for each slice.
-#ifndef SHARED_PROVIDER
-  auto sub_tensor = std::make_unique<Tensor>(tensor_data_type_, per_iteration_shape_,
-                                                     const_cast<void*>(tensor_slice_data_raw), *tensor_location_);
-#else
   auto sub_tensor = Tensor::Create(tensor_data_type_, per_iteration_shape_, const_cast<void*>(tensor_slice_data_raw), *tensor_location_);
-#endif
   auto ml_tensor = DataTypeImpl::GetType<Tensor>();
   current_ = OrtValue{sub_tensor.release(), ml_tensor, ml_tensor->GetDeleteFunc()};
 }
@@ -94,4 +89,3 @@ template class OrtValueTensorSlicer<OrtValue>;
 template class OrtValueTensorSlicer<const OrtValue>;
 
 }  // namespace onnxruntime
-
diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc
index 8092aa32bf..fa44f50f94 100644
--- a/onnxruntime/core/framework/provider_bridge_ort.cc
+++ b/onnxruntime/core/framework/provider_bridge_ort.cc
@@ -156,6 +156,8 @@ struct Node__EdgeIterator_Impl : Node__EdgeIterator {
   Node::EdgeConstIterator v_;
 };
 
+// wrapped = The internal object is exposed as an opaque pointer, so we wrap it in a class that forwards every call to the real calls. No members are ever directly accessed
+// direct = Same implementation is used for shared providers & core code, but some of the methods need to be routed through here to make the linker happy
 struct ProviderHostImpl : ProviderHost {
   void* HeapAllocate(size_t size) override { return new uint8_t[size]; }
   void HeapFree(void* p) override { delete[] reinterpret_cast<uint8_t*>(p); }
@@ -593,8 +595,8 @@ struct ProviderHostImpl : ProviderHost {
   void GraphViewer__operator_delete(GraphViewer* p) override { delete p; }
   std::unique_ptr<Model> GraphViewer__CreateModel(const GraphViewer* graph_viewer, const logging::Logger& logger) override {
     return std::make_unique<Model>(graph_viewer->Name(), true, ModelMetaData(), PathString(),
-                                           IOnnxRuntimeOpSchemaRegistryList(), graph_viewer->DomainToVersionMap(),
-                                           std::vector<ONNX_NAMESPACE::FunctionProto>(), logger);
+                                   IOnnxRuntimeOpSchemaRegistryList(), graph_viewer->DomainToVersionMap(),
+                                   std::vector<ONNX_NAMESPACE::FunctionProto>(), logger);
   }
 
   const std::string& GraphViewer__Name(const GraphViewer* p) noexcept override { return p->Name(); }
@@ -900,7 +902,7 @@ bool InitProvidersSharedLibrary() {
 }
 
 struct ProviderLibrary {
-  ProviderLibrary(const char* filename) : filename_{filename} {}
+  ProviderLibrary(const char* filename, bool unload = true) : filename_{filename}, unload_{unload} {}
   ~ProviderLibrary() {
     assert(!handle_);  // We should already be unloaded at this point
   }
@@ -931,9 +933,9 @@ struct ProviderLibrary {
       if (provider_)
         provider_->Shutdown();
 
-#ifdef _WIN32
-      Env::Default().UnloadDynamicLibrary(handle_);
-#endif
+      if (unload_)
+        Env::Default().UnloadDynamicLibrary(handle_);
+
       handle_ = nullptr;
       provider_ = nullptr;
     }
@@ -941,13 +943,20 @@ struct ProviderLibrary {
 
  private:
   const char* filename_;
+  bool unload_;
   Provider* provider_{};
   void* handle_{};
 
   ORT_DISALLOW_COPY_AND_ASSIGNMENT(ProviderLibrary);
 };
 
-static ProviderLibrary s_library_cuda(LIBRARY_PREFIX "onnxruntime_providers_cuda" LIBRARY_EXTENSION);
+static ProviderLibrary s_library_cuda(LIBRARY_PREFIX "onnxruntime_providers_cuda" LIBRARY_EXTENSION
+#if 0
+#ifndef _WIN32
+                                      ,false /* unload - On Linux if we unload the cuda shared provider we crash */
+#endif
+#endif
+);
 static ProviderLibrary s_library_dnnl(LIBRARY_PREFIX "onnxruntime_providers_dnnl" LIBRARY_EXTENSION);
 static ProviderLibrary s_library_openvino(LIBRARY_PREFIX "onnxruntime_providers_openvino" LIBRARY_EXTENSION);
 static ProviderLibrary s_library_tensorrt(LIBRARY_PREFIX "onnxruntime_providers_tensorrt" LIBRARY_EXTENSION);
@@ -1109,4 +1118,3 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ Or
   options->provider_factories.push_back(factory);
   return nullptr;
 }
-
diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h
index e01dc0e0cc..e81745a917 100644
--- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h
+++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h
@@ -20,9 +20,6 @@
 #include "gsl/gsl"
 
 namespace onnxruntime {
-//class Tensor;
-//class OpKernelContext;
-
 namespace rnn {
 namespace detail {
 
diff --git a/onnxruntime/core/providers/cuda/controlflow/scan.cc b/onnxruntime/core/providers/cuda/controlflow/scan.cc
index d46493d3cb..71caf71611 100644
--- a/onnxruntime/core/providers/cuda/controlflow/scan.cc
+++ b/onnxruntime/core/providers/cuda/controlflow/scan.cc
@@ -7,6 +7,8 @@
 #include "core/providers/cuda/cuda_common.h"
 #include "core/providers/cuda/tensor/transpose.h"
 #include "core/framework/ml_value.h"
+
+// TODO: It's ugly to include a .cc file but this .cc file defines the implementation of some templates which we need.
 #include "core/framework/ort_value_tensor_slicer.cc"
 
 using namespace ONNX_NAMESPACE;
diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h
index bc88e51e13..dc73002a0c 100644
--- a/onnxruntime/core/providers/shared_library/provider_api.h
+++ b/onnxruntime/core/providers/shared_library/provider_api.h
@@ -7,6 +7,7 @@
 //       switching providers to be runnable as shared libraries. The interfaces will become more tightly integrated into the core code.
 
 #pragma once
+// ROCM uses the CUDA provider's files, which are shared provider files. This 'fakes them out' and makes them be non shared provider files if they're being built as part of ROCM.
 #ifdef USE_ROCM
 #include "core/providers/common.h"
 #include "core/providers/cpu/tensor/onehot.h"
diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
index c3f4679318..2661af935a 100644
--- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
+++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
@@ -16,12 +16,6 @@
 #define _Post_writable_byte_size_(n)
 #endif
 
-#if 0
-namespace onnxruntime {
-ProviderHost* g_host = Provider_GetHost();
-}
-#endif
-
 #ifdef _WIN32
 // Override default new/delete so that we match the host's allocator
 _Ret_notnull_ _Post_writable_byte_size_(n) void* operator new(size_t n) { return Provider_GetHost()->HeapAllocate(n); }
@@ -31,6 +25,8 @@ void operator delete(void* p, size_t /*size*/) noexcept { return Provider_GetHos
 
 namespace onnxruntime {
 
+ProviderHost* g_host = Provider_GetHost();
+
 static std::unique_ptr<std::vector<std::function<void()>>> s_run_on_unload_;
 
 void RunOnUnload(std::function<void()> function) {
@@ -71,31 +67,31 @@ AllocatorPtr AllocatorManager::GetAllocator(int id, OrtMemType mem_type) const {
 }
 
 template <>
-MLDataType DataTypeImpl::GetType<Tensor>() { return g_host->DataTypeImpl__GetType_Tensor(); }
+MLDataType DataTypeImpl::GetType<Tensor>() { return Provider_GetHost()->DataTypeImpl__GetType_Tensor(); }
 template <>
-MLDataType DataTypeImpl::GetType<float>() { return g_host->DataTypeImpl__GetType_float(); }
+MLDataType DataTypeImpl::GetType<float>() { return Provider_GetHost()->DataTypeImpl__GetType_float(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<bool>() { return g_host->DataTypeImpl__GetTensorType_bool(); }
+MLDataType DataTypeImpl::GetTensorType<bool>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_bool(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<int8_t>() { return g_host->DataTypeImpl__GetTensorType_int8(); }
+MLDataType DataTypeImpl::GetTensorType<int8_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_int8(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<uint8_t>() { return g_host->DataTypeImpl__GetTensorType_uint8(); }
+MLDataType DataTypeImpl::GetTensorType<uint8_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_uint8(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<int16_t>() { return g_host->DataTypeImpl__GetTensorType_int16(); }
+MLDataType DataTypeImpl::GetTensorType<int16_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_int16(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<uint16_t>() { return g_host->DataTypeImpl__GetTensorType_uint16(); }
+MLDataType DataTypeImpl::GetTensorType<uint16_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_uint16(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<int32_t>() { return g_host->DataTypeImpl__GetTensorType_int32(); }
+MLDataType DataTypeImpl::GetTensorType<int32_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_int32(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<uint32_t>() { return g_host->DataTypeImpl__GetTensorType_uint32(); }
+MLDataType DataTypeImpl::GetTensorType<uint32_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_uint32(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<int64_t>() { return g_host->DataTypeImpl__GetTensorType_int64(); }
+MLDataType DataTypeImpl::GetTensorType<int64_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_int64(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<uint64_t>() { return g_host->DataTypeImpl__GetTensorType_uint64(); }
+MLDataType DataTypeImpl::GetTensorType<uint64_t>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_uint64(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<float>() { return g_host->DataTypeImpl__GetTensorType_float(); }
+MLDataType DataTypeImpl::GetTensorType<float>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_float(); }
 template <>
-MLDataType DataTypeImpl::GetTensorType<double>() { return g_host->DataTypeImpl__GetTensorType_double(); }
+MLDataType DataTypeImpl::GetTensorType<double>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_double(); }
 template <>
 MLDataType DataTypeImpl::GetTensorType<BFloat16>() { return Provider_GetHost()->DataTypeImpl__GetTensorType_BFloat16(); }
 template <>
diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h
index 730be890a8..e33a2ae12e 100644
--- a/onnxruntime/core/providers/shared_library/provider_interfaces.h
+++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h
@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 // Public wrappers around internal ort interfaces (currently)
-// In the future the internal implementations could derive from these to remove the need for the wrapper implementations
 #include "core/providers/shared_library/provider_host_api.h"
 
 #include "core/providers/shared/common.h"
@@ -768,11 +767,10 @@ struct ProviderHost {
 #endif
 };
 
-#define g_host Provider_GetHost()
-//extern ProviderHost* g_host;
-
 #ifdef SHARED_PROVIDER
 
+extern ProviderHost* g_host;
+
 struct CPUIDInfo final {
   static const CPUIDInfo& GetCPUIDInfo() { return g_host->CPUIDInfo__GetCPUIDInfo(); }
 
diff --git a/onnxruntime/test/framework/TestAllocatorManager.cc b/onnxruntime/test/framework/TestAllocatorManager.cc
index ac1722979e..6b32ce1618 100644
--- a/onnxruntime/test/framework/TestAllocatorManager.cc
+++ b/onnxruntime/test/framework/TestAllocatorManager.cc
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
-#undef USE_CUDA  // DO NOT CHECKIN: Temporary until I can patch this up
+
+#undef USE_CUDA  // TODO: Cuda is a shared library, so can't call any Cuda provider methods directly from here
 
 #include "test/framework/TestAllocatorManager.h"
 #include "core/framework/allocatormgr.h"
diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc
index ff1a0269e3..e1e73d160b 100644
--- a/onnxruntime/test/framework/inference_session_test.cc
+++ b/onnxruntime/test/framework/inference_session_test.cc
@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
-#undef USE_CUDA
+#undef USE_CUDA  // TODO: Cuda is a shared library, so can't call any Cuda provider methods directly from here
 
 #include "core/graph/onnx_protobuf.h"
 #include "core/session/inference_session.h"
@@ -166,8 +166,8 @@ static void CreateMatMulModel(std::unique_ptr<onnxruntime::Model>& p_model, Prov
   // Generate the input & output def lists
   std::vector<ONNX_NAMESPACE::FunctionProto> model_specific_functions;
   p_model = std::make_unique<Model>("test", true, ModelMetaData(), PathString(),
-                                            IOnnxRuntimeOpSchemaRegistryList(), domain_to_version,
-                                            model_specific_functions, DefaultLoggingManager().DefaultLogger());
+                                    IOnnxRuntimeOpSchemaRegistryList(), domain_to_version,
+                                    model_specific_functions, DefaultLoggingManager().DefaultLogger());
   onnxruntime::Graph& graph = p_model->MainGraph();
 
   TypeProto tensor_float;
@@ -349,8 +349,8 @@ void RunModelWithBindingMatMul(InferenceSession& session_object,
     auto& shape = rtensor.Shape();
     auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault);
     std::unique_ptr<Tensor> cpu_tensor = std::make_unique<Tensor>(element_type,
-                                                                          shape,
-                                                                          cpu_allocator);
+                                                                  shape,
+                                                                  cpu_allocator);
 #ifdef USE_CUDA
     cudaStream_t stream = static_cast<cudaStream_t>(static_cast<const onnxruntime::CUDAExecutionProvider*>(TestCudaExecutionProvider())->GetComputeStream());
 #elif USE_ROCM
@@ -646,7 +646,7 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) {
   ASSERT_TRUE(size > 1);
   ASSERT_TRUE(lines[0].find("[") != string::npos);
   ASSERT_TRUE(lines[1].find("model_loading_uri") != string::npos);
-  ASSERT_TRUE(lines[size-1].find("]") != string::npos);
+  ASSERT_TRUE(lines[size - 1].find("]") != string::npos);
   std::vector<std::string> tags = {"pid", "dur", "ts", "ph", "X", "name", "args"};
 
   bool has_kernel_info = false;
@@ -867,9 +867,9 @@ static void TestBindHelper(const std::string& log_str,
     epi.device_id = 0;
     EXPECT_TRUE(session_object.RegisterExecutionProvider(std::make_unique<CUDAExecutionProvider>(epi)).IsOK());
 #elif USE_ROCM
-  ROCMExecutionProviderInfo epi;
-  epi.device_id = 0;
-  EXPECT_TRUE(session_object.RegisterExecutionProvider(std::make_unique<ROCMExecutionProvider>(epi)).IsOK());
+    ROCMExecutionProviderInfo epi;
+    epi.device_id = 0;
+    EXPECT_TRUE(session_object.RegisterExecutionProvider(std::make_unique<ROCMExecutionProvider>(epi)).IsOK());
 #endif
   }
 
@@ -1283,19 +1283,18 @@ TEST(ExecutionProviderTest, ShapeInferenceForFusedFunctionTest) {
 
   Graph& fused_graph = session.GetMutableGraph();
   ASSERT_TRUE(fused_graph.NumberOfNodes() == 1);
-  auto &fused_node = *fused_graph.Nodes().begin();
+  auto& fused_node = *fused_graph.Nodes().begin();
   ASSERT_TRUE(fused_node.NodeType() == Node::Type::Fused);
   ASSERT_TRUE(fused_node.Op()->has_type_and_shape_inference_function());
 
   // Clear shape inference data from output node to verify that assigned inference function is called
-  auto &fused_node_output = *fused_node.MutableOutputDefs()[0];
+  auto& fused_node_output = *fused_node.MutableOutputDefs()[0];
   fused_node_output.ClearShape();
   fused_graph.SetGraphResolveNeeded();
   fused_graph.Resolve();
 
   ASSERT_TRUE(fused_node_output.Shape() != nullptr);
-  ASSERT_TRUE(utils::GetTensorShapeFromTensorShapeProto(*fused_node_output.Shape())
-              == utils::GetTensorShapeFromTensorShapeProto(float_tensor.tensor_type().shape()));
+  ASSERT_TRUE(utils::GetTensorShapeFromTensorShapeProto(*fused_node_output.Shape()) == utils::GetTensorShapeFromTensorShapeProto(float_tensor.tensor_type().shape()));
 }
 
 TEST(InferenceSessionTests, Test3LayerNestedSubgraph) {
diff --git a/onnxruntime/test/framework/memcpy_transformer_test.cc b/onnxruntime/test/framework/memcpy_transformer_test.cc
index 836ec7b071..7b08f9df2c 100644
--- a/onnxruntime/test/framework/memcpy_transformer_test.cc
+++ b/onnxruntime/test/framework/memcpy_transformer_test.cc
@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
-#undef USE_CUDA
+#undef USE_CUDA  // TODO: Cuda is a shared library, so can't call any Cuda provider methods directly from here
 
 #include <iterator>
 
diff --git a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc
index f590d1aafd..e54095a5b7 100644
--- a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc
+++ b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc
@@ -262,8 +262,7 @@ TEST(GradientGraphBuilderTest, TrainingSession_WithGist) {
   std::cout << "Loaded " << model_metadata->graph_name << '\n';
 
   // Add cuda execution provider for gist encode/decode nodes
-  CUDAExecutionProviderInfo xp_info;
-  ASSERT_STATUS_OK(training_session.RegisterExecutionProvider(std::make_unique<CUDAExecutionProvider>(xp_info)));
+  ASSERT_STATUS_OK(training_session.RegisterExecutionProvider(DefaultCudaExecutionProvider()));
 
   ORT_THROW_IF_ERROR(training_session.Initialize());
 
diff --git a/orttraining/orttraining/training_ops/cuda/gist/gist.cc b/orttraining/orttraining/training_ops/cuda/gist/gist.cc
index 1b53081792..1591bb8f58 100644
--- a/orttraining/orttraining/training_ops/cuda/gist/gist.cc
+++ b/orttraining/orttraining/training_ops/cuda/gist/gist.cc
@@ -8,14 +8,14 @@ namespace onnxruntime {
 namespace cuda {
 
 // Pack Binary
-#define REGISTER_KERNEL_TYPED_BIN_ENC(T)                                        \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistBinarizeEncoder,                                                      \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_BIN_ENC(T)                                                   \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistBinarizeEncoder,                                                                 \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistBinarizeEncoderOp<T>);
 
 REGISTER_KERNEL_TYPED_BIN_ENC(float)
@@ -38,14 +38,14 @@ Status GistBinarizeEncoderOp<T>::ComputeInternal(OpKernelContext* context) const
   return Status::OK();
 }
 
-#define REGISTER_KERNEL_TYPED_BIN_DEC(T)                                        \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistBinarizeDecoder,                                                      \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_BIN_DEC(T)                                                   \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistBinarizeDecoder,                                                                 \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistBinarizeDecoderOp<T>);
 
 REGISTER_KERNEL_TYPED_BIN_DEC(float)
@@ -69,14 +69,14 @@ Status GistBinarizeDecoderOp<T>::ComputeInternal(OpKernelContext* context) const
 }
 
 // Pack1
-#define REGISTER_KERNEL_TYPED_PACK1_ENC(T)                                      \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPack1Encoder,                                                         \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACK1_ENC(T)                                                 \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPack1Encoder,                                                                    \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPack1EncoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACK1_ENC(bool)
@@ -99,14 +99,14 @@ Status GistPack1EncoderOp<T>::ComputeInternal(OpKernelContext* context) const {
   return Status::OK();
 }
 
-#define REGISTER_KERNEL_TYPED_PACK1_DEC(T)                                      \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPack1Decoder,                                                         \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACK1_DEC(T)                                                 \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPack1Decoder,                                                                    \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPack1DecoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACK1_DEC(bool)
@@ -129,14 +129,14 @@ Status GistPack1DecoderOp<T>::ComputeInternal(OpKernelContext* context) const {
 }
 
 // Pack 8
-#define REGISTER_KERNEL_TYPED_PACK8_ENC(T)                                      \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPack8Encoder,                                                         \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACK8_ENC(T)                                                 \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPack8Encoder,                                                                    \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPack8EncoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACK8_ENC(float)
@@ -160,14 +160,14 @@ Status GistPack8EncoderOp<T>::ComputeInternal(OpKernelContext* context) const {
   return Status::OK();
 }
 
-#define REGISTER_KERNEL_TYPED_PACK8_DEC(T)                                      \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPack8Decoder,                                                         \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACK8_DEC(T)                                                 \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPack8Decoder,                                                                    \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPack8DecoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACK8_DEC(float)
@@ -191,14 +191,14 @@ Status GistPack8DecoderOp<T>::ComputeInternal(OpKernelContext* context) const {
 }
 
 // Pack 16
-#define REGISTER_KERNEL_TYPED_PACK16_ENC(T)                                     \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPack16Encoder,                                                        \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACK16_ENC(T)                                                \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPack16Encoder,                                                                   \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPack16EncoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACK16_ENC(float)
@@ -221,14 +221,14 @@ Status GistPack16EncoderOp<T>::ComputeInternal(OpKernelContext* context) const {
   return Status::OK();
 }
 
-#define REGISTER_KERNEL_TYPED_PACK16_DEC(T)                                     \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPack16Decoder,                                                        \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACK16_DEC(T)                                                \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPack16Decoder,                                                                   \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPack16DecoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACK16_DEC(float)
@@ -251,14 +251,14 @@ Status GistPack16DecoderOp<T>::ComputeInternal(OpKernelContext* context) const {
 }
 
 // Pack MSFP15
-#define REGISTER_KERNEL_TYPED_PACKMSFP15_ENC(T)                                 \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPackMsfp15Encoder,                                                    \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACKMSFP15_ENC(T)                                            \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPackMsfp15Encoder,                                                               \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPackMsfp15EncoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACKMSFP15_ENC(float)
@@ -291,14 +291,14 @@ Status GistPackMsfp15EncoderOp<T>::ComputeInternal(OpKernelContext* context) con
   return Status::OK();
 }
 
-#define REGISTER_KERNEL_TYPED_PACKMSFP15_DEC(T)                                 \
-  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                \
-      GistPackMsfp15Decoder,                                                    \
-      kMSDomain,                                                                \
-      1,                                                                        \
-      T,                                                                        \
-      kCudaExecutionProvider,                                                   \
-      KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
+#define REGISTER_KERNEL_TYPED_PACKMSFP15_DEC(T)                                            \
+  ONNX_OPERATOR_TYPED_KERNEL_EX(                                                           \
+      GistPackMsfp15Decoder,                                                               \
+      kMSDomain,                                                                           \
+      1,                                                                                   \
+      T,                                                                                   \
+      kCudaExecutionProvider,                                                              \
+      (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
       GistPackMsfp15DecoderOp<T>);
 
 REGISTER_KERNEL_TYPED_PACKMSFP15_DEC(float)
diff --git a/orttraining/orttraining/training_ops/cuda/gist/gist.h b/orttraining/orttraining/training_ops/cuda/gist/gist.h
index c125444e8d..55d55499d9 100644
--- a/orttraining/orttraining/training_ops/cuda/gist/gist.h
+++ b/orttraining/orttraining/training_ops/cuda/gist/gist.h
@@ -3,8 +3,6 @@
 
 #pragma once
 
-#include "core/common/common.h"
-#include "core/framework/op_kernel.h"
 #include "core/providers/cuda/cuda_common.h"
 #include "core/providers/cuda/cuda_kernel.h"