diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake
index 746ec55d47..fcff63788b 100644
--- a/cmake/onnxruntime_python.cmake
+++ b/cmake/onnxruntime_python.cmake
@@ -380,6 +380,7 @@ file(GLOB onnxruntime_python_datasets_data CONFIGURE_DEPENDS
 set(onnxruntime_mobile_util_srcs
     ${REPO_ROOT}/tools/python/util/check_onnx_model_mobile_usability.py
     ${REPO_ROOT}/tools/python/util/convert_onnx_models_to_ort.py
+    ${REPO_ROOT}/tools/python/util/file_utils.py
     ${REPO_ROOT}/tools/python/util/logger.py
     ${REPO_ROOT}/tools/python/util/make_dynamic_shape_fixed.py
     ${REPO_ROOT}/tools/python/util/onnx_model_utils.py
@@ -397,6 +398,9 @@ file(GLOB onnxruntime_mobile_helpers_srcs CONFIGURE_DEPENDS
     ${REPO_ROOT}/tools/ci_build/github/android/nnapi_supported_ops.md
     ${REPO_ROOT}/tools/ci_build/github/apple/coreml_supported_ops.md
 )
+file(GLOB onnxruntime_qdq_helper_srcs CONFIGURE_DEPENDS
+    ${REPO_ROOT}/tools/python/util/qdq_helpers/*.py
+)
 
 set(build_output_target onnxruntime_common)
 if(NOT onnxruntime_ENABLE_STATIC_ANALYSIS)
@@ -408,6 +412,7 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/datasets
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/mobile_helpers
+  COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/qdq_helpers
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/ort_format_model
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/ort_format_model/ort_flatbuffers_py
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/transformers
@@ -460,7 +465,17 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E copy
       ${onnxruntime_mobile_util_srcs}
       $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/
-      COMMAND ${CMAKE_COMMAND} -E copy
+  # append the /tools/python/utils imports to the __init__.py that came from /onnxruntime/tools.
+  # we're aggregating scripts from two different locations, and only include selected functionality from
+  # /tools/python/util. due to that we take the full __init__.py from /onnxruntime/tools and append
+  # the required content from /tools/python/util/__init__append.py.
+  COMMAND ${CMAKE_COMMAND} -E cat
+      ${REPO_ROOT}/tools/python/util/__init__append.py >>
+      $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/__init__.py
+  COMMAND ${CMAKE_COMMAND} -E copy
+      ${onnxruntime_qdq_helper_srcs}
+      $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/qdq_helpers/
+  COMMAND ${CMAKE_COMMAND} -E copy
       ${onnxruntime_mobile_helpers_srcs}
       $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/mobile_helpers/
   COMMAND ${CMAKE_COMMAND} -E copy
diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
index 457d20f9a1..70fd33b2c4 100644
--- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
+++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
@@ -54,6 +54,7 @@ static const char* const kOrtSessionOptionsDisableQuantQDQ = "session.disable_qu
 // other factors like whether the model was created using Quantization Aware Training or Post Training Quantization.
 // As such, it's best to test to determine if enabling this works well for your scenario.
 // The default value is "0"
+// Available since version 1.11.
 static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enable_quant_qdq_cleanup";
 
 // Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0".
@@ -80,25 +81,18 @@ static const char* const kOrtSessionOptionsConfigUseORTModelBytesDirectly = "ses
 
 // This should only be specified when exporting an ORT format model for use on a different platform.
 // If the ORT format model will be used on ARM platforms set to "1". For other platforms set to "0"
+// Available since version 1.11.
 static const char* const kOrtSessionOptionsQDQIsInt8Allowed = "session.qdqisint8allowed";
 
-// Save information for replaying graph optimizations later instead of applying them directly.
-//
-// When an ONNX model is loaded, ORT can perform various optimizations on the graph.
-// However, when an ORT format model is loaded, the logic to perform these optimizations may not be available because
-// this scenario must be supported by minimal builds.
-// When loading an ONNX model, ORT can optionally save the effects of some optimizations for later replay in an ORT
-// format model. These are known as "runtime optimizations" - in an ORT format model, they happen at runtime.
-//
-// Note: This option is only applicable when loading an ONNX model and saving an ORT format model.
-//
-// Note: Runtime optimizations are only supported for certain optimizations at the extended level or higher.
-// Unsupported optimizations at those levels are not applied at all, while optimizations at other levels are applied
-// directly.
-//
-// "0": disabled, "1": enabled
-// The default is "0".
-static const char* const kOrtSessionOptionsConfigSaveRuntimeOptimizations = "optimization.save_runtime_optimizations";
+// Specifies how minimal build graph optimizations are handled in a full build.
+// These optimizations are at the extended level or higher.
+// Possible values and their effects are:
+// "save": Save runtime optimizations when saving an ORT format model.
+// "apply": Only apply optimizations available in a minimal build.
+// ""/<unspecified>: Apply optimizations available in a full build.
+// Available since version 1.11.
+static const char* const kOrtSessionOptionsConfigMinimalBuildOptimizations =
+    "optimization.minimal_build_optimizations";
 
 // Note: The options specific to an EP should be specified prior to appending that EP to the session options object in
 // order for them to take effect.
diff --git a/onnxruntime/core/framework/config_options.cc b/onnxruntime/core/framework/config_options.cc
index 05ab8627dd..d74989bb47 100644
--- a/onnxruntime/core/framework/config_options.cc
+++ b/onnxruntime/core/framework/config_options.cc
@@ -22,8 +22,8 @@ bool ConfigOptions::TryGetConfigEntry(const std::string& config_key, std::string
   return found;
 }
 
-const std::string ConfigOptions::GetConfigOrDefault(const std::string& config_key,
-                                                    const std::string& default_value) const noexcept {
+std::string ConfigOptions::GetConfigOrDefault(const std::string& config_key,
+                                              const std::string& default_value) const noexcept {
   return GetConfigEntry(config_key).value_or(default_value);
 }
 
diff --git a/onnxruntime/core/framework/config_options.h b/onnxruntime/core/framework/config_options.h
index 24f682bdad..e70261797f 100644
--- a/onnxruntime/core/framework/config_options.h
+++ b/onnxruntime/core/framework/config_options.h
@@ -12,9 +12,9 @@
 namespace onnxruntime {
 
 /**
-  * Configuration options that can be used by any struct by inheriting this class.
-  * Provides infrastructure to add/get config entries
-  */
+ * Configuration options that can be used by any struct by inheriting this class.
+ * Provides infrastructure to add/get config entries
+ */
 struct ConfigOptions {
   std::unordered_map<std::string, std::string> configurations;
 
@@ -29,7 +29,7 @@ struct ConfigOptions {
 
   // Get the config string in this instance of ConfigOptions using the given config_key
   // If there is no such config, the given default string will be returned
-  const std::string GetConfigOrDefault(const std::string& config_key, const std::string& default_value) const noexcept;
+  std::string GetConfigOrDefault(const std::string& config_key, const std::string& default_value) const noexcept;
 
   // Add a config pair (config_key, config_value) to this instance of ConfigOptions
   Status AddConfigEntry(const char* config_key, const char* config_value) noexcept;
diff --git a/onnxruntime/core/framework/fallback_cpu_capability.cc b/onnxruntime/core/framework/fallback_cpu_capability.cc
index f12598cd58..46b825a149 100644
--- a/onnxruntime/core/framework/fallback_cpu_capability.cc
+++ b/onnxruntime/core/framework/fallback_cpu_capability.cc
@@ -38,12 +38,12 @@ static bool IsSmallInitializer(const onnxruntime::GraphViewer& graph, const Node
 }
 }  // namespace
 
-InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
-                                               const std::string& provider_type,
-                                               gsl::span<const KernelRegistry* const> kernel_registries,
-                                               gsl::span<const NodeIndex> tentative_nodes) {
+std::unordered_set<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
+                                                   const std::string& provider_type,
+                                                   gsl::span<const KernelRegistry* const> kernel_registries,
+                                                   gsl::span<const NodeIndex> tentative_nodes) {
   // automatic conversion from const std::vector&
-  gsl::span<const NodeIndex> ordered_nodes = graph.GetNodesInTopologicalOrder();
+  const auto& ordered_nodes = graph.GetNodesInTopologicalOrder();
   InlinedVector<size_t> node_id_to_order_map(graph.MaxNodeIndex());
   for (size_t id = 0, limit = ordered_nodes.size(); id < limit; ++id) {
     const NodeIndex& node_id = ordered_nodes[id];
@@ -55,7 +55,7 @@ InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& g
     return node_id_to_order_map[n1] > node_id_to_order_map[n2];
   };
 
-  std::priority_queue<NodeIndex, InlinedVector<NodeIndex>, decltype(greater_order_comp)> candidates(greater_order_comp);
+  std::priority_queue<NodeIndex, std::vector<NodeIndex>, decltype(greater_order_comp)> candidates(greater_order_comp);
 
   InlinedHashSet<const NodeArg*> cpu_output_args;
 
@@ -95,10 +95,10 @@ InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& g
         }));
   }
 
-  gsl::span<const NodeArg* const> graph_inputs = graph.GetInputs();
+  const auto& graph_inputs = graph.GetInputs();
   InlinedHashSet<NodeIndex> visited;
   visited.reserve(candidates.size());
-  InlinedHashSet<NodeIndex> cpu_nodes;
+  std::unordered_set<NodeIndex> cpu_nodes;
   cpu_nodes.reserve(candidates.size());
   // The algo below is trying to identity a subgraph that only depends on cpu tensors.
   // Usually it is a subgraph that doing shape calculation based on a GPU tensor, then reshape it back.
diff --git a/onnxruntime/core/framework/fallback_cpu_capability.h b/onnxruntime/core/framework/fallback_cpu_capability.h
index fda5fdb188..531c1c0746 100644
--- a/onnxruntime/core/framework/fallback_cpu_capability.h
+++ b/onnxruntime/core/framework/fallback_cpu_capability.h
@@ -18,9 +18,9 @@ namespace onnxruntime {
   @param kernel_registries Kernel registries for the target EP
   @param tentative_nodes Nodes that are tentative to be placed on on target EP
   */
-InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const GraphViewer& graph,
-                                               const std::string& provider_type,
-                                               gsl::span<const KernelRegistry* const> kernel_registries,
-                                               gsl::span<const NodeIndex> tentative_nodes);
+  std::unordered_set<NodeIndex> GetCpuPreferredNodes(const GraphViewer& graph,
+                                                    const std::string& provider_type,
+                                                    gsl::span<const KernelRegistry* const> kernel_registries,
+                                                    gsl::span<const NodeIndex> tentative_nodes);
 
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc
index 139cbd6d22..7f1499e2e3 100644
--- a/onnxruntime/core/framework/session_state.cc
+++ b/onnxruntime/core/framework/session_state.cc
@@ -988,23 +988,18 @@ Status SessionState::LoadFromOrtFormat(const fbs::SessionState& fbs_session_stat
   // kernel hashes for model are in top level SessionState
   const auto& compiled_kernel_hashes = GetCompiledKernelHashes();
 
-  const bool original_nodes_should_exist =
-      compiled_kernel_hashes.empty()
-#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
-      && graph_.RuntimeOptimizationReplayCtx().num_replayed_optimizations == 0
-#endif  // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
-      ;
-
   // process the nodes that existed when the model was created
   for (FbsSessionStateViewer::Index i = 0, end = fbs_session_state_viewer.GetNumNodeKernelInfos(); i < end; ++i) {
     const auto node_kernel_info = fbs_session_state_viewer.GetNodeKernelInfo(i);
 
     Node* const node = graph_.GetNode(node_kernel_info.node_index);
     if (node == nullptr) {
-      // this is OK if we have compiled kernels/replayed runtime optimizations and the original node was replaced.
+#if defined(ORT_MINIMAL_BUILD) && !defined(ORT_EXTENDED_MINIMAL_BUILD)
+      // this is OK if we have compiled kernels and the original node was replaced.
       // if not the model is invalid.
-      ORT_RETURN_IF(original_nodes_should_exist,
+      ORT_RETURN_IF(compiled_kernel_hashes.empty(),
                     "Can't find node with index ", node_kernel_info.node_index, ". Invalid ORT format model.");
+#endif  // defined(ORT_MINIMAL_BUILD) && !defined(ORT_EXTENDED_MINIMAL_BUILD)
       continue;
     }
 
diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc
index 21216dedc1..67d81a21cb 100644
--- a/onnxruntime/core/optimizer/graph_transformer_utils.cc
+++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc
@@ -286,7 +286,7 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformersForMinimalB
     const IExecutionProvider& cpu_execution_provider,
     const InlinedHashSet<std::string>& rules_and_transformers_to_disable) {
   InlinedVector<std::unique_ptr<GraphTransformer>> transformers;
-  bool saving = std::holds_alternative<SatRuntimeOptimizationSaveContext>(apply_context);
+  const bool saving = std::holds_alternative<SatRuntimeOptimizationSaveContext>(apply_context);
 
   switch (level) {
     case TransformerLevel::Level1:
diff --git a/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api.h b/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api.h
index efd2a7c1b8..91b3986602 100644
--- a/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api.h
+++ b/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api.h
@@ -428,7 +428,7 @@ class GraphRef {
 }  // namespace api
 
 constexpr int64_t kMinSupportedOpset = 7;
-constexpr int64_t kMaxSupportedOpset = 15;
+constexpr int64_t kMaxSupportedOpset = 16;
 
 enum class OptimizerMode {
   OPTIMIZE_TRANSPOSE,        // simple transpose optimization
@@ -441,6 +441,11 @@ enum class OptimizerMode {
 /// <returns>const reference to an unordered set of op_types which are layout sensitive</returns>
 const std::unordered_set<std::string_view>& GetLayoutSensitiveOps();
 
+struct OptimizeResult {
+  std::optional<std::string> error_msg;  // set if there was an error
+  bool graph_modified{false};
+};
+
 /// <summary>
 /// Performs transpose optimization on a graph. Returns true if the graph was modified.
 ///
@@ -453,15 +458,17 @@ const std::unordered_set<std::string_view>& GetLayoutSensitiveOps();
 /// <param name="graph">The graph to optimize (or a portion of a graph, see api::GraphRef docs)</param>
 /// <param name="allow_extended_ops">Whether com.microsoft ops can be used for optimization</param>
 /// <param name="provider_type">Execution provider if applicable.</param>
-/// <param name="mode">Current mode. Optimizer can be called in the context of transpose optimizations or during layout transformations.</param>
-/// <param name="layout_sensitive_ops">List of ops which are treated as layout sensitive by the ONNX standard as well as any runtime specific ops.
-/// These ops should be provided when mode is set to OPTIMIZE_LAYOUT_TRANSFORM. If these ops are not provided, transpose optimizer may convert the
-/// layout for these ops </param>
-/// <returns>true if the graph was modified</returns>
-bool Optimize(api::GraphRef& graph, bool allow_extended_ops,
-              const std::string& provider_type = "",
-              OptimizerMode mode = OptimizerMode::OPTIMIZE_TRANSPOSE,
-              const std::unordered_set<std::string_view>& layout_sensitive_ops = {});
+/// <param name="mode">Current mode. Optimizer can be called in the context of transpose optimizations or during
+/// layout transformations.</param>
+/// <param name="layout_sensitive_ops">List of ops which are treated as layout sensitive by the ONNX standard
+/// as well as any runtime specific ops. These ops should be provided when mode is set to OPTIMIZE_LAYOUT_TRANSFORM.
+/// If these ops are not provided, transpose optimizer may convert the layout for these ops </param>
+/// <returns>OptimizeResult. If error_msg is set the Optimize failed. If not set, graph_modified indicates whether
+/// any changes were required during optimization.</returns>
+OptimizeResult Optimize(api::GraphRef& graph, bool allow_extended_ops,
+                        const std::string& provider_type = "",
+                        OptimizerMode mode = OptimizerMode::OPTIMIZE_TRANSPOSE,
+                        const std::unordered_set<std::string_view>& layout_sensitive_ops = {});
 
 /* Layout Transformation Tools
  * These methods help change the channel ordering of layout sensitive ops (like Conv). ONNX currently only supports
diff --git a/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc b/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc
index 6af96a6263..26e99574b1 100644
--- a/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc
+++ b/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc
@@ -876,9 +876,14 @@ Status TransformLayoutForCompilingEP(Graph& graph, bool& modified, const IExecut
   }
 
   if (modified) {
-    onnx_layout_transformation::Optimize(*api_graph, /*allow_extended_ops*/ true, execution_provider.Type(),
-                                         onnx_layout_transformation::OptimizerMode::OPTIMIZE_LAYOUT_TRANSFORM,
-                                         layout_sensitive_ops);
+    OptimizeResult result =
+        onnx_layout_transformation::Optimize(*api_graph, /*allow_extended_ops*/ true, execution_provider.Type(),
+                                             onnx_layout_transformation::OptimizerMode::OPTIMIZE_LAYOUT_TRANSFORM,
+                                             layout_sensitive_ops);
+    if (result.error_msg) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Optimization after layout transformation failed: ",
+                             result.error_msg.value());
+    }
   }
 
   return Status::OK();
diff --git a/onnxruntime/core/optimizer/transpose_optimizer/ort_transpose_optimizer.cc b/onnxruntime/core/optimizer/transpose_optimizer/ort_transpose_optimizer.cc
index 582842b3ea..bd370a49e0 100644
--- a/onnxruntime/core/optimizer/transpose_optimizer/ort_transpose_optimizer.cc
+++ b/onnxruntime/core/optimizer/transpose_optimizer/ort_transpose_optimizer.cc
@@ -17,7 +17,14 @@ namespace onnxruntime {
 
 Status TransposeOptimizer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
   auto api_graph = MakeApiGraph(graph, cpu_allocator_, /*new_node_ep*/ nullptr);
-  if (onnx_layout_transformation::Optimize(*api_graph, /*allow_extended_ops*/ false)) {
+  OptimizeResult result = onnx_layout_transformation::Optimize(*api_graph, /*allow_extended_ops*/ false);
+  if (result.error_msg) {
+    // currently onnx_layout_transformation::Optimize only fails if we hit an unsupported opset.
+    // we don't want to fail loading the model just because we can't optimize Transpose ops, so just log a warning
+    LOGS(logger, WARNING) << "Transpose optimizer failed: " << result.error_msg.value();
+  }
+
+  if (result.graph_modified) {
     modified = true;
   }
 
diff --git a/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc b/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc
index a46a82272d..bd7ad7efcc 100644
--- a/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc
+++ b/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc
@@ -967,35 +967,41 @@ static void PermuteInput(api::GraphRef& graph, api::NodeRef& node, size_t i, con
   node.SetInput(i, gather_output);
 }
 
-static bool HandleResize(HandlerArgs& args) {
-  auto inputs = args.node.Inputs();
-  int64_t rank_int = gsl::narrow_cast<int64_t>(args.perm.size());
+// static bool HandleResize(HandlerArgs& args) {
+//  auto inputs = args.node.Inputs();
+//  int64_t rank_int = gsl::narrow_cast<int64_t>(args.perm.size());
+//
+//  auto p = ChannelFirstToLastPerm(rank_int);
+//  auto& perm = p == args.perm ? args.perm : args.perm_inv;
+//  auto& perm_inv = p == args.perm ? args.perm_inv : args.perm;
+//
+//  if (args.ctx.opset < 11) {
+//     PermuteInput(args.ctx.graph, args.node, 1, perm);
+//   } else {
+//     if (inputs[1] != "") {
+//       std::vector<int64_t> double_perm_inv = perm;
+//       double_perm_inv.reserve(2 * args.perm.size());
+//       for (int64_t p1 : perm) {
+//         double_perm_inv.push_back(p1 + rank_int);
+//       }
+//       PermuteInput(args.ctx.graph, args.node, 1, double_perm_inv);
+//     }
+//     for (size_t i = 2; i < inputs.size(); ++i) {
+//       if (inputs[i] != "") {
+//         PermuteInput(args.ctx.graph, args.node, i, perm);
+//       }
+//     }
+//   }
+//
+//   TransposeFirstInput(args.ctx, args.node, perm);
+//   TransposeOutputs(args.ctx, args.node, perm_inv);
+//
+//   SwapNodeOpTypeAndDomain(args.ctx.graph, args.node, args.node.OpType(), "com.microsoft.nhwc");
+//
+//   return true;
+// }
 
-  if (args.ctx.opset < 11) {
-    PermuteInput(args.ctx.graph, args.node, 1, args.perm_inv);
-  } else {
-    if (inputs[1] != "") {
-      std::vector<int64_t> double_perm_inv = args.perm_inv;
-      double_perm_inv.reserve(2 * args.perm_inv.size());
-      for (int64_t p : args.perm_inv) {
-        double_perm_inv.push_back(p + rank_int);
-      }
-      PermuteInput(args.ctx.graph, args.node, 1, double_perm_inv);
-    }
-    for (size_t i = 2; i < inputs.size(); ++i) {
-      if (inputs[i] != "") {
-        PermuteInput(args.ctx.graph, args.node, i, args.perm_inv);
-      }
-    }
-  }
-
-  TransposeFirstInput(args.ctx, args.node, args.perm_inv);
-  TransposeOutputs(args.ctx, args.node, args.perm);
-
-  return true;
-}
-
-constexpr HandlerInfo resize_handler = {&FirstInput, &HandleResize};
+// constexpr HandlerInfo resize_handler = {&FirstInput, &HandleResize};
 
 static bool HandlePad(HandlerArgs& args) {
   size_t rank = args.perm.size();
@@ -1412,7 +1418,7 @@ static bool HandleTile(HandlerArgs& args) {
 
 constexpr HandlerInfo tile_handler = {&FirstInput, &HandleTile};
 
-// Helper to remove cancelling Transpose -> Transpose or 
+// Helper to remove cancelling Transpose -> Transpose or
 // Transpose -> Reshape nodes.
 static void RemoveCancelingTransposeNodes(HandlerArgs& args) {
   // Input to 1st transpose
@@ -1491,7 +1497,7 @@ static bool HandleTranspose(HandlerArgs& args) {
 constexpr HandlerInfo transpose_handler = {&FirstInput, &HandleTranspose, /*transposes_outputs*/ false};
 
 static bool HandleReshape(HandlerArgs& args) {
-  // We check for a very specific case where Transpose is replaced by Reshape 
+  // We check for a very specific case where Transpose is replaced by Reshape
   // for performance. For example Transpose(input {1, 1, 1, X}, perm{0, 3, 2, 1}) can be replaced by Reshape
   // Reshape(input{1, 1, 1, X}, shape{1, X, 1, 1})
   // During transpose optimization we need to detect such reshape nodes so that we can remove them if possible.
@@ -1502,7 +1508,7 @@ static bool HandleReshape(HandlerArgs& args) {
     return false;
   }
 
-  // Check only 1 dim is not equal to 1. This is to validate that tranpose and reshape are truly canceling nodes 
+  // Check only 1 dim is not equal to 1. This is to validate that tranpose and reshape are truly canceling nodes
   // and can be therefore removed.
   int num_dims_not_equal_to_1 = 0;
   for (int i = 0; i < 4; i++) {
@@ -1521,7 +1527,7 @@ static bool HandleReshape(HandlerArgs& args) {
   }
 
   // Check whether transpose cancels with reshape node
-  // We check if shape of transpose node's input matches the shape data 
+  // We check if shape of transpose node's input matches the shape data
   // provided for reshape node.
   auto reshape_output_shape = DataInt64(*shape_data);
   if (reshape_output_shape != transpose_input_shape) {
@@ -1691,7 +1697,9 @@ static const std::unordered_map<std::string_view, const HandlerInfo&> handler_ma
     {"Split", split_handler},
     {"Shape", shape_handler},
     {"Pad", pad_handler},
-    {"Resize", resize_handler},
+    // Todo: renable resize handler after adding NHWC support in upsample op on cpu
+    // https://github.com/microsoft/onnxruntime/issues/9857
+    //  {"Resize", resize_handler},
     {"ReduceSum", reduce_sum_handler},
 
     {"ReduceLogSum", reduce_op_handler},
@@ -1812,14 +1820,22 @@ bool ProcessTranspose(OptimizerCtx& ctx, api::NodeRef& transpose, api::NodeRef&
 // Returns nullopt if graph opset is unsupported.
 std::optional<OptimizerCtx> MakeOptimizerContext(api::GraphRef& graph, bool allow_extended_ops,
                                                  const std::string& provider_type, OptimizerMode mode,
-                                                 const std::unordered_set<std::string_view>& layout_sensitive_ops) {
+                                                 const std::unordered_set<std::string_view>& layout_sensitive_ops,
+                                                 std::string& error_msg) {
   auto opset = graph.Opset("");
   if (opset == std::nullopt) {
     opset = graph.Opset("ai.onnx");
   }
+
   if (opset == std::nullopt || *opset > kMaxSupportedOpset || *opset < kMinSupportedOpset) {
+    // if the model doesn't have an ONNX opset that's fine as there are no ops we'd move around
+    if (opset.has_value()) {
+      error_msg = "Unsupported ONNX opset";
+    }
+
     return std::nullopt;
   }
+
   if (allow_extended_ops) {
     auto ms_opset = graph.Opset("com.microsoft");
     if (ms_opset == std::nullopt || *ms_opset != 1) {
@@ -1836,7 +1852,8 @@ std::optional<OptimizerCtx> MakeOptimizerContext(api::GraphRef& graph, bool allo
 
 // Performs optimization. General algorithm: iterate over nodes in topological order. If a node has a transpose
 // as input, push it through if the transpose cost does not increase and is likely to decrease.
-bool OptimizeImpl(OptimizerCtx& ctx) {
+OptimizeResult OptimizeImpl(OptimizerCtx& ctx) {
+  OptimizeResult result{};
   const std::vector<std::unique_ptr<api::NodeRef>> nodes = ctx.graph.Nodes();
 
   std::unordered_set<std::string> outputs_leading_to_transpose;
@@ -1901,7 +1918,8 @@ bool OptimizeImpl(OptimizerCtx& ctx) {
   // Currently limiting the second optimization pass to layout transform mode
   // TODO: Enable this for both the modes.
   if (ctx.mode == OptimizerMode::OPTIMIZE_TRANSPOSE) {
-    return changed;
+    result.graph_modified = changed;
+    return result;
   }
 
   // Run second optimization pass.
@@ -1944,25 +1962,35 @@ bool OptimizeImpl(OptimizerCtx& ctx) {
       changed = true;
     }
   }
-  return changed;
+
+  result.graph_modified = changed;
+  return result;
 }
 
 const std::unordered_set<std::string_view>& GetLayoutSensitiveOps() {
   // List of all layout sensitive ops defined in ONNX standard.
   static std::unordered_set<std::string_view> layout_sensitive_ops = {"Conv", "QLinearConv", "BatchNormalization",
                                                                       "AveragePool", "GlobalAveragePool", "MaxPool",
-                                                                      "GlobalMaxPool", "LRN"};
+                                                                      "GlobalMaxPool", "LRN", "GridSample"};
 
   return layout_sensitive_ops;
 }
 
-bool Optimize(api::GraphRef& graph, bool allow_extended_ops,
-              const std::string& provider_type, OptimizerMode mode,
-              const std::unordered_set<std::string_view>& layout_sensitive_ops) {
-  auto ctx = MakeOptimizerContext(graph, allow_extended_ops, provider_type, mode, layout_sensitive_ops);
+OptimizeResult Optimize(api::GraphRef& graph, bool allow_extended_ops,
+                        const std::string& provider_type, OptimizerMode mode,
+                        const std::unordered_set<std::string_view>& layout_sensitive_ops) {
+  OptimizeResult result{};
+
+  std::string error_msg;
+  auto ctx = MakeOptimizerContext(graph, allow_extended_ops, provider_type, mode, layout_sensitive_ops, error_msg);
   if (ctx == std::nullopt) {
-    return false;
+    if (!error_msg.empty()) {
+      result.error_msg = error_msg;
+    }
+
+    return result;
   }
+
   return OptimizeImpl(*ctx);
 }
 
diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc
index 52aa9cd02a..1a3818b758 100644
--- a/onnxruntime/core/platform/windows/env.cc
+++ b/onnxruntime/core/platform/windows/env.cc
@@ -544,22 +544,21 @@ class WindowsEnv : public Env {
 #endif
     if (!*handle) {
       const auto error_code = GetLastError();
-      LPVOID lpMsgBuf;
+      static constexpr DWORD bufferLength = 64 * 1024;
+      std::wstring s(bufferLength, '\0');
       FormatMessageW(
-          FORMAT_MESSAGE_ALLOCATE_BUFFER |
               FORMAT_MESSAGE_FROM_SYSTEM |
               FORMAT_MESSAGE_IGNORE_INSERTS,
           NULL,
           error_code,
           MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-          (LPWSTR)&lpMsgBuf,
+          (LPWSTR)s.data(),
           0, NULL);
       std::wostringstream oss;
-      oss << L"LoadLibrary failed with error " << error_code << L" \"" << (LPWSTR)lpMsgBuf << L"\" when trying to load \"" << wlibrary_filename << L"\"";
+      oss << L"LoadLibrary failed with error " << error_code << L" \"" << s.c_str() << L"\" when trying to load \"" << wlibrary_filename << L"\"";
       std::wstring errmsg = oss.str();
       // TODO: trim the ending '\r' and/or '\n'
       common::Status status(common::ONNXRUNTIME, common::FAIL, ToUTF8String(errmsg));
-      LocalFree(lpMsgBuf);
       return status;
     }
     return Status::OK();
@@ -577,22 +576,21 @@ class WindowsEnv : public Env {
     *symbol = ::GetProcAddress(reinterpret_cast<HMODULE>(handle), symbol_name.c_str());
     if (!*symbol) {
       const auto error_code = GetLastError();
-      LPVOID lpMsgBuf;
+      static constexpr DWORD bufferLength = 64 * 1024;
+      std::wstring s(bufferLength, '\0');
       FormatMessageW(
-          FORMAT_MESSAGE_ALLOCATE_BUFFER |
               FORMAT_MESSAGE_FROM_SYSTEM |
               FORMAT_MESSAGE_IGNORE_INSERTS,
           NULL,
           error_code,
           MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-          (LPWSTR)&lpMsgBuf,
+          (LPWSTR)s.data(),
           0, NULL);
       std::wostringstream oss;
-      oss << L"Failed to find symbol " << ToWideString(symbol_name) << L" in library, error code: " << error_code << L" \"" << (LPWSTR)lpMsgBuf << L"\"";
+      oss << L"Failed to find symbol " << ToWideString(symbol_name) << L" in library, error code: " << error_code << L" \"" << s.c_str() << L"\"";
       std::wstring errmsg = oss.str();
       // TODO: trim the ending '\r' and/or '\n'
       common::Status status(common::ONNXRUNTIME, common::FAIL, ToUTF8String(errmsg));
-      LocalFree(lpMsgBuf);
       return status;
     }
     return Status::OK();
diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.cc b/onnxruntime/core/providers/cpu/tensor/upsample.cc
index dd292b6e83..61e4d28cf0 100644
--- a/onnxruntime/core/providers/cpu/tensor/upsample.cc
+++ b/onnxruntime/core/providers/cpu/tensor/upsample.cc
@@ -420,15 +420,13 @@ struct BilinearParams {
 // that amounts to 'Bilinear' Upsampling/Resizing in the sense that it assumes
 // the scale values for the outermost 2 dimensions are 1.
 // This is the common use-case where the 4-D input (batched multi-channel images)
-// is usually of shapes:
-// - [N, C, H, W] and the scales are [1.0, 1.0, height_scale, width_scale]
-// - [N, H, W, C] and the scales are [1.0, height_scale, width_scale, 1.0]
-static BilinearParams SetupUpsampleBilinear(const int64_t input_height,
-                                            const int64_t input_width,
-                                            const int64_t output_height,
-                                            const int64_t output_width,
-                                            const float height_scale,
-                                            const float width_scale,
+// is usually of shape [N, C, H, W] and the scales are [1.0, 1.0, height_scale, width_scale]
+static BilinearParams SetupUpsampleBilinear(int64_t input_height,
+                                            int64_t input_width,
+                                            int64_t output_height,
+                                            int64_t output_width,
+                                            float height_scale,
+                                            float width_scale,
                                             const std::vector<float>& roi,
                                             AllocatorPtr& alloc,
                                             const GetOriginalCoordinateFunc& get_original_coordinate) {
@@ -525,25 +523,26 @@ static BilinearParams SetupUpsampleBilinear(const int64_t input_height,
 }
 
 template <typename T>
-void UpsampleBilinear(const int64_t batch_size,
-                      const int64_t num_channels,
-                      const int64_t input_height,
-                      const int64_t input_width,
-                      const int64_t output_height,
-                      const int64_t output_width,
-                      const float height_scale,
-                      const float width_scale,
+void UpsampleBilinear(int64_t batch_size,
+                      int64_t num_channels,
+                      int64_t input_height,
+                      int64_t input_width,
+                      int64_t output_height,
+                      int64_t output_width,
+                      float height_scale,
+                      float width_scale,
                       const std::vector<float>& roi,
-                      const bool use_extrapolation,
-                      const float extrapolation_value,
-                      const T* const XdataBase,
-                      T* const YdataBase,
+                      bool use_extrapolation,
+                      float extrapolation_value,
+                      const T* XdataBase,
+                      T* YdataBase,
                       AllocatorPtr& alloc,
                       const GetOriginalCoordinateFunc& get_original_coordinate,
                       concurrency::ThreadPool* tp) {
   BilinearParams p = SetupUpsampleBilinear(input_height, input_width, output_height, output_width,
                                            height_scale, width_scale, roi,
                                            alloc, get_original_coordinate);
+
   for (int64_t n = 0; n < batch_size; ++n) {
     concurrency::ThreadPool::TrySimpleParallelFor(
         tp, num_channels,
@@ -1066,65 +1065,22 @@ Status Upsample<T>::BaseCompute(OpKernelContext* context,
     case UpsampleMode::LINEAR: {
       // Supports 'bilinear' and 'trilinear' sampling only
 
-      //'bilinear' == 2-D input or 4-D input with outermost 2 scales as 1 or
-      // 4-D input with outermost and innermost scales as 1
+      //'bilinear' == 2-D input or 4-D input with outermost 2 scales as 1
       if (dims.size() == 2 || dims.size() == 4) {
         bool is_2D = dims.size() == 2;
 
-        int64_t batch_size;
-        int64_t num_channels;
-        int64_t input_height;
-        int64_t input_width;
+        const int64_t batch_size = is_2D ? 1 : dims[0];
+        const int64_t num_channels = is_2D ? 1 : dims[1];
+        const int64_t input_height = is_2D ? dims[0] : dims[2];
+        const int64_t input_width = is_2D ? dims[1] : dims[3];
 
-        int64_t output_height;
-        int64_t output_width;
-
-        float height_scale;
-        float width_scale;
-
-        if (is_2D) {
-          batch_size = 1;
-          num_channels = 1;
-          input_height = dims[0];
-          input_width = dims[1];
-
-          output_height = output_dims[0];
-          output_width = output_dims[1];
-
-          height_scale = scales[0];
-          width_scale = scales[1];
-        } else {
-          if (scales[1] == 1.0f) {
-            batch_size = dims[0];
-            num_channels = dims[1];
-            input_height = dims[2];
-            input_width = dims[3];
-
-            output_height = output_dims[2];
-            output_width = output_dims[3];
-
-            height_scale = scales[2];
-            width_scale = scales[3];
-          } else {
-            ORT_ENFORCE(scales[3] == 1.0f, "4-D input with innermost scale (usually channel of NHWC) as 1.");
-
-            batch_size = dims[0];
-            num_channels = dims[3];
-            input_height = dims[1];
-            input_width = dims[2];
-
-            output_height = output_dims[1];
-            output_width = output_dims[2];
-
-            height_scale = scales[1];
-            width_scale = scales[2];
-          }
-        }
+        const int64_t output_height = is_2D ? output_dims[0] : output_dims[2];
+        const int64_t output_width = is_2D ? output_dims[1] : output_dims[3];
 
         AllocatorPtr alloc;
         ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
         UpsampleBilinear(batch_size, num_channels, input_height, input_width, output_height, output_width,
-                         height_scale, width_scale, roi,
+                         is_2D ? scales[0] : scales[2], is_2D ? scales[1] : scales[3], roi,
                          use_extrapolation_, extrapolation_value_, X->Data<T>(),
                          Y->MutableData<T>(), alloc, get_original_coordinate_,
                          output_height * output_width > 64 ? context->GetOperatorThreadPool() : nullptr);
diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc
index 48d6b3dd33..cfc2080fd9 100644
--- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc
+++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc
@@ -7,6 +7,7 @@
 #include "core/providers/cuda/cuda_provider_options.h"
 
 #include <memory>
+#include <chrono>
 
 #include "gsl/gsl"
 
@@ -187,6 +188,26 @@ struct CUDA_Provider : Provider {
   void* GetInfo() override { return &g_info; }
 
   std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory(const void* void_params) override {
+
+    // Calling a function like ::cudaDeviceSynchronize will cause CUDA to ensure there is binary code for the current GPU architecture
+    // Ideally this will be already part of the binary, but if not, CUDA will JIT it during this call. This can take a very long time
+    // (minutes even), so we want to detect when this happens and let the user know why so they can report it properly or even fix it.
+    // See the linked issue in the warning message for more info
+    {
+      auto start_time = std::chrono::steady_clock::now();
+      // Do a trivial cuda operation that will cause JIT to occur
+      {
+        void** cuda_memory {};
+        ::cudaMalloc(&cuda_memory, 1);
+        ::cudaFree(cuda_memory);
+      }
+      auto end_time = std::chrono::steady_clock::now();
+      auto duration = std::chrono::duration_cast<std::chrono::seconds>(end_time - start_time);
+      if (duration > std::chrono::seconds{30}) {
+        LOGS_DEFAULT(WARNING) << "CUDA took " << duration.count() << " seconds to start, please see this issue for how to fix it: https://github.com/microsoft/onnxruntime/issues/10746";
+      }
+    }
+
     auto params = reinterpret_cast<const OrtCUDAProviderOptionsV2*>(void_params);
 
     CUDAExecutionProviderInfo info{};
diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h
index 9688559795..efc55f600a 100644
--- a/onnxruntime/core/providers/shared_library/provider_api.h
+++ b/onnxruntime/core/providers/shared_library/provider_api.h
@@ -251,10 +251,10 @@ std::unique_ptr<IAllocator> CreateROCMPinnedAllocator(int16_t device_id, const c
 
 std::unique_ptr<IDataTransfer> CreateGPUDataTransfer(void* stream);
 
-InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
-                                               const std::string& provider_type,
-                                               gsl::span<const KernelRegistry* const> kernel_registries,
-                                               gsl::span<const NodeIndex> tentative_nodes);
+std::unordered_set<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
+                                                   const std::string& provider_type,
+                                                   gsl::span<const KernelRegistry* const> kernel_registries,
+                                                   gsl::span<const NodeIndex> tentative_nodes);
 
 std::string GetEnvironmentVar(const std::string& var_name);
 
diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
index eb6ebb390a..ccbcd47cd1 100644
--- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
+++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc
@@ -347,10 +347,10 @@ std::string GetEnvironmentVar(const std::string& var_name) {
   return g_host->GetEnvironmentVar(var_name);
 }
 
-InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
-                                               const std::string& provider_type,
-                                               gsl::span<const KernelRegistry* const> kernel_registries,
-                                               gsl::span<const NodeIndex> tentative_nodes) {
+std::unordered_set<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
+                                                   const std::string& provider_type,
+                                                   gsl::span<const KernelRegistry* const> kernel_registries,
+                                                   gsl::span<const NodeIndex> tentative_nodes) {
   return g_host->GetCpuPreferredNodes(graph, provider_type, kernel_registries, tentative_nodes);
 }
 
diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h
index 72dc98ef53..a9f85fe2b5 100644
--- a/onnxruntime/core/providers/shared_library/provider_interfaces.h
+++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h
@@ -173,10 +173,10 @@ struct ProviderHost {
   virtual bool RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0;
 #endif
 
-  virtual InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
-                                                         const std::string& provider_type,
-                                                         gsl::span<const KernelRegistry* const> kernel_registries,
-                                                         gsl::span<const NodeIndex> tentative_nodes) = 0;
+  virtual std::unordered_set<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
+                                                             const std::string& provider_type,
+                                                             gsl::span<const KernelRegistry* const> kernel_registries,
+                                                             gsl::span<const NodeIndex> tentative_nodes) = 0;
 
   virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ bool* p_data, size_t expected_size) = 0;
   virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ float* p_data, size_t expected_size) = 0;
diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
index 2876486ef8..92f08c486d 100644
--- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
+++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
@@ -251,6 +251,11 @@ TensorrtLogger& GetTensorrtLogger() {
   return trt_logger;
 }
 
+std::unique_lock<OrtMutex> TensorrtExecutionProvider::GetApiLock() const {
+  static OrtMutex singleton;
+  return std::unique_lock<OrtMutex>(singleton);
+}
+
 TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProviderInfo& info)
     : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider, true}, info_(info), device_id_(info.device_id) {
   CUDA_CALL_THROW(cudaSetDevice(device_id_));
@@ -396,7 +401,10 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
         throw std::runtime_error("Failed to create directory " + cache_path_);
       }
     }
-    runtime_ = tensorrt_ptr::unique_pointer<nvinfer1::IRuntime>(nvinfer1::createInferRuntime(GetTensorrtLogger()));
+    {
+      auto lock = GetApiLock();
+      runtime_ = tensorrt_ptr::unique_pointer<nvinfer1::IRuntime>(nvinfer1::createInferRuntime(GetTensorrtLogger()));
+    }
   }
 
   if (engine_decryption_enable_) {
@@ -1001,13 +1009,6 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
   return result;
 }
 
-std::unique_lock<OrtMutex> TensorrtExecutionProvider::GetEngineBuildLock() const {
-  static OrtMutex singleton;
-
-  // Acquire a lock only when force_sequential_engine_build_ is true;
-  return force_sequential_engine_build_ ? std::unique_lock<OrtMutex>(singleton) : std::unique_lock<OrtMutex>();
-}
-
 common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fused_nodes,
                                                   std::vector<NodeComputeInfo>& node_compute_funcs) {
   for (const auto* fused_node : fused_nodes) {
@@ -1197,7 +1198,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
 
         // Build engine
         {
-          auto lock = GetEngineBuildLock();
+          auto lock = GetApiLock();
           trt_engine = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(trt_builder->buildEngineWithConfig(*trt_network, *trt_config));
         }
         if (trt_engine == nullptr) {
@@ -1538,7 +1539,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
 
         // Build engine
         {
-          auto lock = GetEngineBuildLock();
+          auto lock = GetApiLock();
           *(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(
               trt_builder->buildEngineWithConfig(*trt_state->network->get(), *trt_config));
         }
diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
index ee759228ae..9aaec129c1 100644
--- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
+++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
@@ -194,10 +194,10 @@ class TensorrtExecutionProvider : public IExecutionProvider {
   void RemoveTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const GraphViewer& graph) const;
 
   /** 
-  Get a unique_lock object to control the concurrency behavior of TensorRT engine building. When force_sequential_engine_build
-  is set to true, the lock object is associated with a mutex shared across all providers to enforce sequential engine build. 
-  Otherwise, the constructed unique_lock is not associated with any mutex therefore no locking/unlocking will happen.
+  Get a unique_lock object to control the concurrency behavior. 
+  Every api call not in the thread-safe operations(https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#threading)
+  should be protected by a lock when invoked by multiple threads concurrently.
   */
-  std::unique_lock<OrtMutex> GetEngineBuildLock() const;
+  std::unique_lock<OrtMutex> GetApiLock() const;
 };
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc
index 0fe1972ab8..e0f3e6a373 100644
--- a/onnxruntime/core/session/inference_session.cc
+++ b/onnxruntime/core/session/inference_session.cc
@@ -163,10 +163,10 @@ Status VerifyEachNodeIsAssignedToAnEp(const Graph& graph, const logging::Logger&
 
   return status;
 }
-}  // namespace
 
 #if !defined(ORT_MINIMAL_BUILD)
-static bool AreAllNodesInMainGraphAssignedToOneEp(const Graph& graph, ProviderType provider) {
+
+bool AreAllNodesInMainGraphAssignedToOneEp(const Graph& graph, ProviderType provider) {
   for (const auto& node : graph.Nodes()) {
     const auto& node_provider = node.GetExecutionProviderType();
 
@@ -178,7 +178,7 @@ static bool AreAllNodesInMainGraphAssignedToOneEp(const Graph& graph, ProviderTy
   return true;
 }
 
-static bool HasControlflowNodes(const Graph& graph) {
+bool HasControlflowNodes(const Graph& graph) {
   for (const auto& node : graph.Nodes()) {
     if (node.ContainsSubgraph()) {
       return true;
@@ -187,7 +187,40 @@ static bool HasControlflowNodes(const Graph& graph) {
 
   return false;
 }
-#endif
+
+Status GetMinimalBuildOptimizationHandling(
+    std::string_view config_value, bool saving_ort_format,
+    InferenceSession::MinimalBuildOptimizationHandling& minimal_build_optimization_handling) {
+  if (config_value == "save") {
+    if (saving_ort_format) {
+      minimal_build_optimization_handling =
+          InferenceSession::MinimalBuildOptimizationHandling::SaveMinimalBuildRuntimeOptimizations;
+      return Status::OK();
+    }
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           kOrtSessionOptionsConfigMinimalBuildOptimizations,
+                           " value of 'save' is only valid when saving an ORT format model.");
+  }
+
+  if (config_value == "apply") {
+    minimal_build_optimization_handling =
+        InferenceSession::MinimalBuildOptimizationHandling::OnlyApplyMinimalBuildOptimizations;
+    return Status::OK();
+  }
+
+  if (config_value.empty()) {
+    minimal_build_optimization_handling =
+        InferenceSession::MinimalBuildOptimizationHandling::ApplyFullBuildOptimizations;
+    return Status::OK();
+  }
+
+  return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                         "Invalid value for ", kOrtSessionOptionsConfigMinimalBuildOptimizations, ": ", config_value);
+};
+
+#endif  // !defined(ORT_MINIMAL_BUILD)
+
+}  // namespace
 
 std::atomic<uint32_t> InferenceSession::global_session_id_{1};
 
@@ -1247,6 +1280,9 @@ Status AssignNodesToEpsFromHashesImpl(Graph& graph, const fbs::SessionState& fbs
   for (const auto& node : graph.Nodes()) {
     if (node.GetExecutionProviderType().empty()) {
       auto kernel_hash = utils::GetHashValueFromStaticKernelHashMap(node.OpType(), node.SinceVersion());
+      if (!kernel_hash.has_value()) {
+        kernel_hash = utils::GetInternalNhwcOpHash(node);
+      }
       if (kernel_hash.has_value()) {
         ORT_RETURN_IF_ERROR(set_node_ep(node.Index(), kernel_hash.value()));
       }
@@ -1402,14 +1438,17 @@ common::Status InferenceSession::Initialize() {
 
 #if !defined(ORT_MINIMAL_BUILD)
     if (!loading_ort_format) {
-      const bool saving_runtime_optimizations =
-          saving_ort_format &&
-          session_options_.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigSaveRuntimeOptimizations,
-                                                             "0") == "1";
+      const auto minimal_build_opt_config_value = session_options_.config_options.GetConfigOrDefault(
+          kOrtSessionOptionsConfigMinimalBuildOptimizations, "");
+      MinimalBuildOptimizationHandling minimal_build_optimization_handling{};
+      ORT_RETURN_IF_ERROR_SESSIONID_(GetMinimalBuildOptimizationHandling(minimal_build_opt_config_value,
+                                                                         saving_ort_format,
+                                                                         minimal_build_optimization_handling));
+
       // add predefined transformers
       ORT_RETURN_IF_ERROR_SESSIONID_(AddPredefinedTransformers(graph_transformation_mgr_,
                                                                session_options_.graph_optimization_level,
-                                                               saving_runtime_optimizations));
+                                                               minimal_build_optimization_handling));
 
       // apply any transformations to the main graph and any subgraphs
       ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, graph_transformation_mgr_,
@@ -1436,9 +1475,9 @@ common::Status InferenceSession::Initialize() {
             // Return error status as we don't want the session initialization to complete successfully
             // if the user has requested usage of CUDA Graph feature and we cannot honor that.
             ORT_RETURN_IF_ERROR_SESSIONID_(
-              ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
-                              "This session cannot use the CUDA Graph feature as requested by the user "
-                              " as the model has control flow nodes which can't be supported by CUDA Graphs."));
+                ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                                "This session cannot use the CUDA Graph feature as requested by the user "
+                                " as the model has control flow nodes which can't be supported by CUDA Graphs."));
           } else if (!AreAllNodesInMainGraphAssignedToOneEp(graph, onnxruntime::kCudaExecutionProvider)) {
             LOGS(*session_logger_, ERROR) << "This session cannot use the CUDA Graph feature as requested by the user "
                                           << " as all the graph nodes have not been partitioned to the CUDA EP.";
@@ -1446,9 +1485,9 @@ common::Status InferenceSession::Initialize() {
             // Return error status as we don't want the session initialization to complete successfully
             // if the user has requested usage of CUDA Graph feature and we cannot honor that.
             ORT_RETURN_IF_ERROR_SESSIONID_(
-              ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
-                              "This session cannot use the CUDA Graph feature as requested by the user "
-                              " as all the graph nodes have not been partitioned to the CUDA EP."));
+                ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                                "This session cannot use the CUDA Graph feature as requested by the user "
+                                " as all the graph nodes have not been partitioned to the CUDA EP."));
 
           } else {
             LOGS(*session_logger_, INFO) << "This session will use the CUDA Graph feature as requested by the user.";
@@ -1875,11 +1914,11 @@ Status InferenceSession::Run(const RunOptions& run_options,
 
   // Check if this Run() is simply going to be a CUDA Graph replay.
   if (cached_execution_provider_for_graph_replay_.IsGraphCaptured()) {
-      LOGS(*session_logger_, INFO) << "Replaying the captured "
-                                   << cached_execution_provider_for_graph_replay_.Type()
-                                   << " CUDA Graph for this model with tag: " << run_options.run_tag;
-      ++current_num_runs_;
-      ORT_RETURN_IF_ERROR_SESSIONID_(cached_execution_provider_for_graph_replay_.ReplayGraph());
+    LOGS(*session_logger_, INFO) << "Replaying the captured "
+                                 << cached_execution_provider_for_graph_replay_.Type()
+                                 << " CUDA Graph for this model with tag: " << run_options.run_tag;
+    ++current_num_runs_;
+    ORT_RETURN_IF_ERROR_SESSIONID_(cached_execution_provider_for_graph_replay_.ReplayGraph());
   } else {
     std::vector<IExecutionProvider*> exec_providers_to_stop;
     exec_providers_to_stop.reserve(execution_providers_.NumProviders());
@@ -1951,13 +1990,13 @@ Status InferenceSession::Run(const RunOptions& run_options,
       }
 #endif
 
-    // execute the graph
+      // execute the graph
 #ifdef DEBUG_NODE_INPUTS_OUTPUTS
       session_state_->IncrementGraphExecutionCounter();
 #endif
       ORT_CHECK_AND_SET_RETVAL(utils::ExecuteGraph(*session_state_, feeds_fetches_manager, feeds, *p_fetches,
-                                                  session_options_.execution_mode, run_options.terminate, run_logger,
-                                                  run_options.only_execute_path_to_fetches));
+                                                   session_options_.execution_mode, run_options.terminate, run_logger,
+                                                   run_options.only_execute_path_to_fetches));
     }
     ORT_CATCH(const std::exception& e) {
       ORT_HANDLE_EXCEPTION([&]() {
@@ -2010,7 +2049,7 @@ Status InferenceSession::Run(const RunOptions& run_options,
   // are needed before replaying the captured graph, here run the inference again
   // to capture the graph, so that users just need one session run to capture
   // the graph.
-  if (retval.IsOK() &&  cached_execution_provider_for_graph_replay_.IsGraphCaptureEnabled() &&
+  if (retval.IsOK() && cached_execution_provider_for_graph_replay_.IsGraphCaptureEnabled() &&
       !cached_execution_provider_for_graph_replay_.IsGraphCaptured()) {
     LOGS(*session_logger_, INFO) << "Start the second Run() to capture the graph. "
                                     "The first one is for necessary memory allocation;"
@@ -2361,21 +2400,30 @@ void InferenceSession::InitLogger(logging::LoggingManager* logging_manager) {
 #if !defined(ORT_MINIMAL_BUILD)
 
 // Registers all the predefined transformers with transformer manager
-common::Status InferenceSession::AddPredefinedTransformers(GraphTransformerManager& transformer_manager,
-                                                           TransformerLevel graph_optimization_level,
-                                                           bool saving_runtime_optimizations) const {
+common::Status InferenceSession::AddPredefinedTransformers(
+    GraphTransformerManager& transformer_manager,
+    TransformerLevel graph_optimization_level,
+    MinimalBuildOptimizationHandling minimal_build_optimization_handling) const {
   const auto& cpu_ep = *execution_providers_.Get(onnxruntime::kCpuExecutionProvider);
   for (int i = static_cast<int>(TransformerLevel::Level1); i <= static_cast<int>(TransformerLevel::MaxLevel); i++) {
     TransformerLevel level = static_cast<TransformerLevel>(i);
     if (graph_optimization_level >= level) {
       // Generate and register transformers for level
       auto transformers_to_register = [&]() {
-        if (!saving_runtime_optimizations || level == TransformerLevel::Level1) {
+        const bool use_full_build_optimizations =
+            level == TransformerLevel::Level1 ||
+            minimal_build_optimization_handling == MinimalBuildOptimizationHandling::ApplyFullBuildOptimizations;
+
+        if (use_full_build_optimizations) {
           return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep,
                                                        optimizers_to_disable_);
         } else {
-          SatRuntimeOptimizationSaveContext save_context{kernel_registry_manager_};
-          return optimizer_utils::GenerateTransformersForMinimalBuild(level, session_options_, save_context, cpu_ep,
+          const auto sat_context =
+              minimal_build_optimization_handling ==
+                      MinimalBuildOptimizationHandling::SaveMinimalBuildRuntimeOptimizations
+                  ? SatApplyContextVariant{SatRuntimeOptimizationSaveContext{kernel_registry_manager_}}
+                  : SatApplyContextVariant{SatDirectApplicationContext{}};
+          return optimizer_utils::GenerateTransformersForMinimalBuild(level, session_options_, sat_context, cpu_ep,
                                                                       optimizers_to_disable_);
         }
       }();
diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h
index 1f8b26cbe5..4c4cc884bd 100644
--- a/onnxruntime/core/session/inference_session.h
+++ b/onnxruntime/core/session/inference_session.h
@@ -107,6 +107,23 @@ struct ModelMetadata {
 
 class InferenceSession {
  public:
+#if !defined(ORT_MINIMAL_BUILD)
+
+  /**
+   * How minimal build graph optimizations should be handled in a full build.
+   * Note: These only apply to optimizations at the extended level or higher.
+   */
+  enum class MinimalBuildOptimizationHandling {
+    /** Run full build optimizations. The default behavior. */
+    ApplyFullBuildOptimizations,
+    /** Save minimal build optimizations as runtime optimizations in an ORT format model. */
+    SaveMinimalBuildRuntimeOptimizations,
+    /** Only run minimal build optimizations. */
+    OnlyApplyMinimalBuildOptimizations,
+  };
+
+#endif
+
   /**
     Create a new InferenceSession
     @param session_options Session options.
@@ -444,6 +461,7 @@ class InferenceSession {
 
  protected:
 #if !defined(ORT_MINIMAL_BUILD)
+
   /**
    * Load an ONNX model.
    * @param protobuf object corresponding to the model file. model_proto will be copied by the API.
@@ -583,9 +601,10 @@ class InferenceSession {
   void ShrinkMemoryArenas(const std::vector<AllocatorPtr>& arenas_to_shrink);
 
 #if !defined(ORT_MINIMAL_BUILD)
-  virtual common::Status AddPredefinedTransformers(GraphTransformerManager& transformer_manager,
-                                                   TransformerLevel graph_optimization_level,
-                                                   bool saving_runtime_optimizations) const;
+  virtual common::Status AddPredefinedTransformers(
+      GraphTransformerManager& transformer_manager,
+      TransformerLevel graph_optimization_level,
+      MinimalBuildOptimizationHandling minimal_build_optimization_handling) const;
 
   common::Status TransformGraph(onnxruntime::Graph& graph,
                                 const onnxruntime::GraphTransformerManager& graph_transformer_mgr,
diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc
index 352aafc2f8..dee2e32c50 100644
--- a/onnxruntime/core/session/onnxruntime_c_api.cc
+++ b/onnxruntime/core/session/onnxruntime_c_api.cc
@@ -2500,7 +2500,6 @@ static constexpr OrtApi ort_api_1_to_11 = {
     &OrtApis::GetSparseTensorIndices,
     // End of Version 9 - DO NOT MODIFY ABOVE (see above text for more information)
 
-    // Version 10 - In development, feel free to add/remove/rearrange here
     &OrtApis::HasValue,
     &OrtApis::KernelContext_GetGPUComputeStream,
     &OrtApis::GetTensorMemoryInfo,
@@ -2515,13 +2514,13 @@ static constexpr OrtApi ort_api_1_to_11 = {
     &OrtApis::SynchronizeBoundOutputs,
     // End of Version 10 - DO NOT MODIFY ABOVE (see above text for more information)
 
-    // Version 11 - In development, feel free to add/remove/rearrange here
     &OrtApis::SessionOptionsAppendExecutionProvider_CUDA_V2,
     &OrtApis::CreateCUDAProviderOptions,
     &OrtApis::UpdateCUDAProviderOptions,
     &OrtApis::GetCUDAProviderOptionsAsString,
     &OrtApis::ReleaseCUDAProviderOptions,
     &OrtApis::SessionOptionsAppendExecutionProvider_MIGraphX,
+    // End of Version 11 - DO NOT MODIFY ABOVE (see above text for more information)
 };
 
 // Asserts to do a some checks to ensure older Versions of the OrtApi never change (will detect an addition or deletion but not if they cancel out each other)
@@ -2536,6 +2535,7 @@ static_assert(offsetof(OrtApi, GetCurrentGpuDeviceId) / sizeof(void*) == 161, "S
 static_assert(offsetof(OrtApi, CreateSessionFromArrayWithPrepackedWeightsContainer) / sizeof(void*) == 169, "Size of version 8 API cannot change");
 static_assert(offsetof(OrtApi, GetSparseTensorIndices) / sizeof(void*) == 191, "Size of version 9 API cannot change");
 static_assert(offsetof(OrtApi, SynchronizeBoundOutputs) / sizeof(void*) == 203, "Size of version 10 API cannot change");
+static_assert(offsetof(OrtApi, SessionOptionsAppendExecutionProvider_MIGraphX) / sizeof(void*) == 209, "Size of version 11 API cannot change");
 
 // So that nobody forgets to finish an API version, this check will serve as a reminder:
 static_assert(std::string_view(ORT_VERSION) == "1.11.0", "ORT_Version change detected, please follow below steps to ensure OrtApi is updated properly");
diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc
index f55b56209d..c730472f31 100644
--- a/onnxruntime/core/session/provider_bridge_ort.cc
+++ b/onnxruntime/core/session/provider_bridge_ort.cc
@@ -207,10 +207,10 @@ struct ProviderHostImpl : ProviderHost {
 
   std::string GetEnvironmentVar(const std::string& var_name) override { return Env::Default().GetEnvironmentVar(var_name); }
 
-  InlinedHashSet<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
-                                                 const std::string& provider_type,
-                                                 gsl::span<const KernelRegistry* const> kernel_registries,
-                                                 gsl::span<const NodeIndex> tentative_nodes) override {
+  std::unordered_set<NodeIndex> GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph,
+                                                     const std::string& provider_type,
+                                                     gsl::span<const KernelRegistry* const> kernel_registries,
+                                                     gsl::span<const NodeIndex> tentative_nodes) override {
     return onnxruntime::GetCpuPreferredNodes(graph, provider_type, kernel_registries, tentative_nodes);
   }
 
diff --git a/onnxruntime/python/_pybind_state.py.in b/onnxruntime/python/_pybind_state.py.in
index 4fa859a559..f8bdbd9a59 100644
--- a/onnxruntime/python/_pybind_state.py.in
+++ b/onnxruntime/python/_pybind_state.py.in
@@ -8,16 +8,26 @@ Ensure that dependencies are available and then load the extension module.
 import os
 import platform
 import sys
+import warnings
 
 from . import _ld_preload  # noqa: F401
 
 if platform.system() == "Windows":
     from . import version_info
 
+    # If on Windows, check if this import error is caused by the user not installing the 2019 VC Runtime
+    # The VC Redist installer usually puts the VC Runtime dlls in the System32 folder, but it may also be found
+    # in some other locations.
+    # TODO, we may want to try to load the VC Runtime dlls instead of checking if the hardcoded file path
+    # is valid, and raise ImportError if the load fails
     if version_info.vs2019 and platform.architecture()[0] == "64bit":
-        if not os.path.isfile("C:\\Windows\\System32\\vcruntime140_1.dll"):
-            raise ImportError(
-                "Microsoft Visual C++ Redistributable for Visual Studio 2019 not installed on the machine.")
+        system_root = os.getenv("SystemRoot") or "C:\\Windows"
+        if not os.path.isfile(os.path.join(system_root, "System32", "vcruntime140_1.dll")):
+            warnings.warn("Please install the 2019 Visual C++ runtime and then try again. "
+                          "If you've installed the runtime in a non-standard location "
+                          "(other than %SystemRoot%\System32), "
+                          "make sure it can be found by setting the correct path.")
+
 @ONNXRUNTIME_SETDLOPENFLAGS_GLOBAL@
 from .onnxruntime_pybind11_state import *  # noqa
 @ONNXRUNTIME_SETDLOPENFLAGS_LOCAL@
diff --git a/onnxruntime/python/tools/quantization/CalTableFlatBuffers/KeyValue.py b/onnxruntime/python/tools/quantization/CalTableFlatBuffers/KeyValue.py
index 3d82f2b599..fc5d725db0 100644
--- a/onnxruntime/python/tools/quantization/CalTableFlatBuffers/KeyValue.py
+++ b/onnxruntime/python/tools/quantization/CalTableFlatBuffers/KeyValue.py
@@ -6,17 +6,20 @@ import flatbuffers
 from flatbuffers.compat import import_numpy
 np = import_numpy()
 
-
 class KeyValue(object):
     __slots__ = ['_tab']
 
     @classmethod
-    def GetRootAsKeyValue(cls, buf, offset):
+    def GetRootAs(cls, buf, offset=0):
         n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
         x = KeyValue()
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def GetRootAsKeyValue(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
     # KeyValue
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)
@@ -35,18 +38,19 @@ class KeyValue(object):
             return self._tab.String(o + self._tab.Pos)
         return None
 
-
+def Start(builder): builder.StartObject(2)
 def KeyValueStart(builder):
-    builder.StartObject(2)
-
-
+    """This method is deprecated. Please switch to Start."""
+    return Start(builder)
+def AddKey(builder, key): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(key), 0)
 def KeyValueAddKey(builder, key):
-    builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(key), 0)
-
-
+    """This method is deprecated. Please switch to AddKey."""
+    return AddKey(builder, key)
+def AddValue(builder, value): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0)
 def KeyValueAddValue(builder, value):
-    builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(value), 0)
-
-
+    """This method is deprecated. Please switch to AddValue."""
+    return AddValue(builder, value)
+def End(builder): return builder.EndObject()
 def KeyValueEnd(builder):
-    return builder.EndObject()
+    """This method is deprecated. Please switch to End."""
+    return End(builder)
\ No newline at end of file
diff --git a/onnxruntime/python/tools/quantization/CalTableFlatBuffers/TrtTable.py b/onnxruntime/python/tools/quantization/CalTableFlatBuffers/TrtTable.py
index dcb8372569..b31f4cac1d 100644
--- a/onnxruntime/python/tools/quantization/CalTableFlatBuffers/TrtTable.py
+++ b/onnxruntime/python/tools/quantization/CalTableFlatBuffers/TrtTable.py
@@ -6,17 +6,20 @@ import flatbuffers
 from flatbuffers.compat import import_numpy
 np = import_numpy()
 
-
 class TrtTable(object):
     __slots__ = ['_tab']
 
     @classmethod
-    def GetRootAsTrtTable(cls, buf, offset):
+    def GetRootAs(cls, buf, offset=0):
         n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
         x = TrtTable()
         x.Init(buf, n + offset)
         return x
 
+    @classmethod
+    def GetRootAsTrtTable(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
     # TrtTable
     def Init(self, buf, pos):
         self._tab = flatbuffers.table.Table(buf, pos)
@@ -46,18 +49,19 @@ class TrtTable(object):
         o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
         return o == 0
 
-
+def Start(builder): builder.StartObject(1)
 def TrtTableStart(builder):
-    builder.StartObject(1)
-
-
+    """This method is deprecated. Please switch to Start."""
+    return Start(builder)
+def AddDict(builder, dict): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(dict), 0)
 def TrtTableAddDict(builder, dict):
-    builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(dict), 0)
-
-
+    """This method is deprecated. Please switch to AddDict."""
+    return AddDict(builder, dict)
+def StartDictVector(builder, numElems): return builder.StartVector(4, numElems, 4)
 def TrtTableStartDictVector(builder, numElems):
-    return builder.StartVector(4, numElems, 4)
-
-
+    """This method is deprecated. Please switch to Start."""
+    return StartDictVector(builder, numElems)
+def End(builder): return builder.EndObject()
 def TrtTableEnd(builder):
-    return builder.EndObject()
+    """This method is deprecated. Please switch to End."""
+    return End(builder)
\ No newline at end of file
diff --git a/onnxruntime/python/tools/quantization/calibrate.py b/onnxruntime/python/tools/quantization/calibrate.py
index 18e0accc4c..c559112028 100644
--- a/onnxruntime/python/tools/quantization/calibrate.py
+++ b/onnxruntime/python/tools/quantization/calibrate.py
@@ -599,7 +599,7 @@ class HistogramCollector(CalibrationDataCollector):
 
         print("Number of tensors : {}".format(len(histogram_dict)))
         print("Number of histogram bins : {}".format(self.num_bins))
-        print("Percentile : {}".format(percentile))
+        print("Percentile : ({},{})".format(100.0 - percentile, percentile))
 
         for tensor, histogram in histogram_dict.items():
             hist = histogram[0]
@@ -607,11 +607,12 @@ class HistogramCollector(CalibrationDataCollector):
             total = hist.sum()
             cdf = np.cumsum(hist/total)
             if self.symmetric:
-                idx_right = np.searchsorted(cdf, percentile/100)
-                thresholds_dict[tensor] = (-float(hist_edges[idx_right]), float(hist_edges[idx_right]))
+                idx_right = np.searchsorted(cdf, percentile / 100.0)
+                thresholds_dict[tensor] = (-float(hist_edges[idx_ringht]), float(hist_edges[idx_right]))
             else:
-                idx_right = np.searchsorted(cdf, percentile/200)
-                idx_left = np.searchsorted(cdf, (1.0 - percentile/200))
+                percent_to_cut_one_side = (100.0 - percentile) / 200.0
+                idx_right = np.searchsorted(cdf, 1.0 - percent_to_cut_one_side)
+                idx_left = np.searchsorted(cdf, percent_to_cut_one_side)
                 thresholds_dict[tensor] = (float(hist_edges[idx_left]), float(hist_edges[idx_right]))
 
             # Plot histogram for debug only
diff --git a/onnxruntime/python/tools/quantization/quant_utils.py b/onnxruntime/python/tools/quantization/quant_utils.py
index bb9ec646a7..8cd0814dd3 100644
--- a/onnxruntime/python/tools/quantization/quant_utils.py
+++ b/onnxruntime/python/tools/quantization/quant_utils.py
@@ -393,7 +393,7 @@ def write_calibration_table(calibration_cache):
     TrtTable.TrtTableStartDictVector(builder, len(key_value_list))
     for key_value in key_value_list:
         builder.PrependUOffsetTRelative(key_value)
-    main_dict = builder.EndVector(len(key_value_list))
+    main_dict = builder.EndVector()
 
     TrtTable.TrtTableStart(builder)
     TrtTable.TrtTableAddDict(builder, main_dict)
diff --git a/onnxruntime/test/framework/test_utils.h b/onnxruntime/test/framework/test_utils.h
index 12ae831b3b..1e97f44629 100644
--- a/onnxruntime/test/framework/test_utils.h
+++ b/onnxruntime/test/framework/test_utils.h
@@ -93,6 +93,15 @@ using OpCountMap = std::map<std::string, int>;
 // Helper function to check that the graph transformations have been successfully applied.
 OpCountMap CountOpsInGraph(const Graph& graph, bool recurse_into_subgraphs = true);
 
+// Gets the op count from the OpCountMap.
+// Can be called with a const OpCountMap, unlike OpCountMap::operator[].
+inline int OpCount(const OpCountMap& op_count_map, const std::string& op_type) {
+  if (auto it = op_count_map.find(op_type); it != op_count_map.end()) {
+    return it->second;
+  }
+  return 0;
+}
+
 #if !defined(DISABLE_SPARSE_TENSORS)
 void SparseIndicesChecker(const ONNX_NAMESPACE::TensorProto& indices_proto, gsl::span<const int64_t> expected_indicies);
 #endif // DISABLE_SPARSE_TENSORS
diff --git a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc
index 6b179b49c7..eb01ec687b 100644
--- a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc
+++ b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc
@@ -185,7 +185,7 @@ using GraphCheckerFn = std::function<void(const Graph& graph)>;
 
 void LoadAndInitializeSession(const SessionOptions& so, const PathString& input_model_path,
                               const GraphOpCountsCheckerFn& graph_op_count_checker_fn,
-                              const GraphCheckerFn* graph_checker_fn = nullptr) {
+                              const GraphCheckerFn& graph_checker_fn = {}) {
   InferenceSessionWrapper session{so, GetEnvironment()};
 
   ASSERT_STATUS_OK(session.Load(input_model_path));
@@ -196,10 +196,12 @@ void LoadAndInitializeSession(const SessionOptions& so, const PathString& input_
 
   const auto initialized_ops = CountOpsInGraph(session.GetGraph());
 
-  graph_op_count_checker_fn(loaded_ops, initialized_ops);
+  if (graph_op_count_checker_fn) {
+    graph_op_count_checker_fn(loaded_ops, initialized_ops);
+  }
 
   if (graph_checker_fn) {
-    (*graph_checker_fn)(session.GetGraph());
+    graph_checker_fn(session.GetGraph());
   }
 }
 
@@ -223,7 +225,7 @@ void SaveAndLoadRuntimeOptimizationsForModel(
     if (do_save) {
       SessionOptions so{};
       ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsConfigSaveModelFormat, "ORT"));
-      ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsConfigSaveRuntimeOptimizations, "1"));
+      ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsConfigMinimalBuildOptimizations, "save"));
       so.graph_optimization_level = TransformerLevel::Level2;
       so.optimized_model_filepath = saved_runtime_optimizations_model_path;
 
@@ -296,7 +298,7 @@ void CheckNhwcTransformerIsApplied() {
                     (OpCountMap{{"Transpose", 6},
                                 {"com.microsoft.QLinearConv", n}}));
         },
-        &checker_fn));
+        checker_fn));
   }
 }
 }  // namespace
@@ -341,6 +343,42 @@ TEST(GraphRuntimeOptimizationTest, TestNhwcTransformer) {
   CheckNhwcTransformerIsApplied();
 }
 
+#if !defined(ORT_MINIMAL_BUILD)
+TEST(GraphRuntimeOptimizationTest, TestOnlyApplyMinimalBuildOptimizations) {
+  // This test assumes that AttentionFusion is not included in the minimal build optimizations.
+  // Update it if that changes.
+
+  // When setting the option to only apply minimal build optimizations, verify that AttentionFusion does not run.
+  {
+    SessionOptions so{};
+    ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsConfigMinimalBuildOptimizations, "apply"));
+    so.graph_optimization_level = TransformerLevel::Level2;
+
+    LoadAndInitializeSession(
+        so,
+        ORT_TSTR("testdata/transform/fusion/attention_int32_mask.onnx"),
+        [](const OpCountMap& /*initialized_ops*/, const OpCountMap& loaded_ops) {
+          // expect no fused node
+          EXPECT_EQ(OpCount(loaded_ops, "com.microsoft.Attention"), 0);
+        });
+  }
+
+  // Otherwise, it should run.
+  {
+    SessionOptions so{};
+    so.graph_optimization_level = TransformerLevel::Level2;
+
+    LoadAndInitializeSession(
+        so,
+        ORT_TSTR("testdata/transform/fusion/attention_int32_mask.onnx"),
+        [](const OpCountMap& /*initialized_ops*/, const OpCountMap& loaded_ops) {
+          // expect fused node
+          EXPECT_EQ(OpCount(loaded_ops, "com.microsoft.Attention"), 1);
+        });
+  }
+}
+#endif  // !defined(ORT_MINIMAL_BUILD)
+
 #endif  // !defined(DISABLE_CONTRIB_OPS)
 
 }  // namespace onnxruntime::test
diff --git a/onnxruntime/test/optimizer/transpose_optimizer_test.cc b/onnxruntime/test/optimizer/transpose_optimizer_test.cc
index 176d154158..198bee9f75 100644
--- a/onnxruntime/test/optimizer/transpose_optimizer_test.cc
+++ b/onnxruntime/test/optimizer/transpose_optimizer_test.cc
@@ -291,209 +291,212 @@ TEST(TransposeOptimizerTests, TestPadNonconst) {
                     /*opset_version*/ 11);
 }
 
-TEST(TransposeOptimizerTests, TestResize) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
-    auto* const_1 = builder.MakeInitializer<float>({4}, {0.3f, 2.5f, 1.0f, 0.7f});
-    auto* transpose_1_out_0 = builder.MakeIntermediate();
-    auto* resize_1_out_0 = builder.MakeIntermediate();
-    auto* transpose_2_out_0 = builder.MakeOutput();
+// Todo: renable tests on resize transformer after adding NHWC support in upsample op on cpu
+// https://github.com/microsoft/onnxruntime/issues/9857
 
-    auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    builder.AddNode("Resize", {transpose_1_out_0, const_1}, {resize_1_out_0});
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1,
-                    /*opset_version*/ 10);
-}
-
-TEST(TransposeOptimizerTests, TestResizeOpset11) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
-    auto* const_1 = builder.MakeInitializer<float>({8}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
-    auto* const_2 = builder.MakeInitializer<float>({4}, {0.3f, 2.5f, 1.0f, 0.7f});
-    auto* transpose_1_out_0 = builder.MakeIntermediate();
-    auto* resize_1_out_0 = builder.MakeIntermediate();
-    auto* transpose_2_out_0 = builder.MakeOutput();
-
-    auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    builder.AddNode("Resize", {transpose_1_out_0, const_1, const_2}, {resize_1_out_0});
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1,
-                    /*opset_version*/ 11);
-}
-
-TEST(TransposeOptimizerTests, TestResizeOpset15) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
-    auto* const_1 = builder.MakeInitializer<float>({4}, {0.3f, 2.5f, 1.0f, 0.7f});
-    auto* transpose_1_out_0 = builder.MakeIntermediate();
-    auto* resize_1_out_0 = builder.MakeIntermediate();
-    auto* transpose_2_out_0 = builder.MakeOutput();
-    auto empty_arg = NodeArg("", nullptr);
-
-    auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    builder.AddNode("Resize", {transpose_1_out_0, &empty_arg, const_1}, {resize_1_out_0});
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1,
-                    /*opset_version*/ 15);
-}
-
-TEST(TransposeOptimizerTests, TestResizeSizeRoi) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
-    auto* const_1 = builder.MakeInitializer<float>({8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f});
-    auto* const_2 = builder.MakeInitializer<int64_t>({4}, {10, 9, 8, 7});
-    auto* transpose_1_out_0 = builder.MakeIntermediate();
-    auto* resize_1_out_0 = builder.MakeIntermediate();
-    auto* transpose_2_out_0 = builder.MakeOutput();
-    auto empty_arg = NodeArg("", nullptr);
-
-    auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, const_1, &empty_arg, const_2}, {resize_1_out_0});
-    resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize");
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1,
-                    /*opset_version*/ 15);
-}
-
-TEST(TransposeOptimizerTests, TestResizeRoiScalesZeroRank0) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input = builder.MakeInput<uint8_t>({1, 512, 512, 3},
-                                             std::numeric_limits<uint8_t>::min(),
-                                             std::numeric_limits<uint8_t>::max());
-    auto* resize_in_roi = builder.MakeInitializer<float>({0}, {});
-    auto* resize_in_scales = builder.MakeInitializer<float>({0}, {});
-    auto* resize_in_sizes = builder.MakeInitializer<int64_t>({4}, {1, 256, 32, 32});
-
-    auto* transpose1_out_transposed = builder.MakeIntermediate();
-    auto* resize_out_Y = builder.MakeIntermediate();
-    auto* output = builder.MakeOutput();
-
-    auto& transpose_1 = builder.AddNode("Transpose", {input}, {transpose1_out_transposed});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    builder.AddNode("Resize",
-                    {transpose1_out_transposed, resize_in_roi, resize_in_scales, resize_in_sizes},
-                    {resize_out_Y});
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_out_Y}, {output});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1);
-}
-
-TEST(TransposeOptimizerTests, TestResizeNonconst) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
-    auto* input1_arg = MakeInput<float>(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f});
-    auto* input2_arg = MakeInput<float>(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f});
-    auto* transpose_1_out_0 = builder.MakeIntermediate();
-    auto* resize_1_out_0 = builder.MakeIntermediate();
-    auto* transpose_2_out_0 = builder.MakeOutput();
-
-    auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0});
-    resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize");
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1,
-                    /*opset_version*/ 11);
-}
-
-TEST(TransposeOptimizerTests, TestResizeNonconstOpset13) {
-  auto build_test_case_1 = [&](ModelTestBuilder& builder) {
-    auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
-    auto* input1_arg = MakeInput<float>(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f});
-    auto* input2_arg = MakeInput<float>(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f});
-    auto* transpose_1_out_0 = builder.MakeIntermediate();
-    auto* resize_1_out_0 = builder.MakeIntermediate();
-    auto* transpose_2_out_0 = builder.MakeOutput();
-
-    auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
-    transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
-    auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0});
-    resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize");
-    auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
-    transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
-  };
-
-  auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
-    int transpose_cost = EstimateTransposeCost(session.GetGraph());
-    EXPECT_EQ(transpose_cost, 0);
-  };
-
-  TransformerTester(build_test_case_1,
-                    check_optimized_graph_1,
-                    TransformerLevel::Default,
-                    TransformerLevel::Level1,
-                    /*opset_version*/ 13);
-}
+// TEST(TransposeOptimizerTests, TestResize) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
+//     auto* const_1 = builder.MakeInitializer<float>({4}, {0.3f, 2.5f, 1.0f, 0.7f});
+//     auto* transpose_1_out_0 = builder.MakeIntermediate();
+//     auto* resize_1_out_0 = builder.MakeIntermediate();
+//     auto* transpose_2_out_0 = builder.MakeOutput();
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     builder.AddNode("Resize", {transpose_1_out_0, const_1}, {resize_1_out_0});
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1,
+//                     /*opset_version*/ 10);
+// }
+//
+// TEST(TransposeOptimizerTests, TestResizeOpset11) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
+//     auto* const_1 = builder.MakeInitializer<float>({8}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
+//     auto* const_2 = builder.MakeInitializer<float>({4}, {0.3f, 2.5f, 1.0f, 0.7f});
+//     auto* transpose_1_out_0 = builder.MakeIntermediate();
+//     auto* resize_1_out_0 = builder.MakeIntermediate();
+//     auto* transpose_2_out_0 = builder.MakeOutput();
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     builder.AddNode("Resize", {transpose_1_out_0, const_1, const_2}, {resize_1_out_0});
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1,
+//                     /*opset_version*/ 11);
+// }
+//
+// TEST(TransposeOptimizerTests, TestResizeOpset15) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
+//     auto* const_1 = builder.MakeInitializer<float>({4}, {0.3f, 2.5f, 1.0f, 0.7f});
+//     auto* transpose_1_out_0 = builder.MakeIntermediate();
+//     auto* resize_1_out_0 = builder.MakeIntermediate();
+//     auto* transpose_2_out_0 = builder.MakeOutput();
+//     auto empty_arg = NodeArg("", nullptr);
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     builder.AddNode("Resize", {transpose_1_out_0, &empty_arg, const_1}, {resize_1_out_0});
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1,
+//                     /*opset_version*/ 15);
+// }
+//
+// TEST(TransposeOptimizerTests, TestResizeSizeRoi) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
+//     auto* const_1 = builder.MakeInitializer<float>({8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f});
+//     auto* const_2 = builder.MakeInitializer<int64_t>({4}, {10, 9, 8, 7});
+//     auto* transpose_1_out_0 = builder.MakeIntermediate();
+//     auto* resize_1_out_0 = builder.MakeIntermediate();
+//     auto* transpose_2_out_0 = builder.MakeOutput();
+//     auto empty_arg = NodeArg("", nullptr);
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, const_1, &empty_arg, const_2}, {resize_1_out_0});
+//     resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize");
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1,
+//                     /*opset_version*/ 15);
+// }
+//
+// TEST(TransposeOptimizerTests, TestResizeRoiScalesZeroRank0) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input = builder.MakeInput<uint8_t>({1, 512, 512, 3},
+//                                              std::numeric_limits<uint8_t>::min(),
+//                                              std::numeric_limits<uint8_t>::max());
+//     auto* resize_in_roi = builder.MakeInitializer<float>({0}, {});
+//     auto* resize_in_scales = builder.MakeInitializer<float>({0}, {});
+//     auto* resize_in_sizes = builder.MakeInitializer<int64_t>({4}, {1, 256, 32, 32});
+//
+//     auto* transpose1_out_transposed = builder.MakeIntermediate();
+//     auto* resize_out_Y = builder.MakeIntermediate();
+//     auto* output = builder.MakeOutput();
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input}, {transpose1_out_transposed});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     builder.AddNode("Resize",
+//                     {transpose1_out_transposed, resize_in_roi, resize_in_scales, resize_in_sizes},
+//                     {resize_out_Y});
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_out_Y}, {output});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1);
+// }
+//
+// TEST(TransposeOptimizerTests, TestResizeNonconst) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
+//     auto* input1_arg = MakeInput<float>(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f});
+//     auto* input2_arg = MakeInput<float>(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f});
+//     auto* transpose_1_out_0 = builder.MakeIntermediate();
+//     auto* resize_1_out_0 = builder.MakeIntermediate();
+//     auto* transpose_2_out_0 = builder.MakeOutput();
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0});
+//     resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize");
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1,
+//                     /*opset_version*/ 11);
+// }
+//
+// TEST(TransposeOptimizerTests, TestResizeNonconstOpset13) {
+//   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
+//     auto* input0_arg = MakeInput<float>(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0);
+//     auto* input1_arg = MakeInput<float>(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f});
+//     auto* input2_arg = MakeInput<float>(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f});
+//     auto* transpose_1_out_0 = builder.MakeIntermediate();
+//     auto* resize_1_out_0 = builder.MakeIntermediate();
+//     auto* transpose_2_out_0 = builder.MakeOutput();
+//
+//     auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0});
+//     transpose_1.AddAttribute("perm", std::vector<int64_t>{0, 3, 1, 2});
+//     auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0});
+//     resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize");
+//     auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0});
+//     transpose_2.AddAttribute("perm", std::vector<int64_t>{0, 2, 3, 1});
+//   };
+//
+//   auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) {
+//     int transpose_cost = EstimateTransposeCost(session.GetGraph());
+//     EXPECT_EQ(transpose_cost, 0);
+//   };
+//
+//   TransformerTester(build_test_case_1,
+//                     check_optimized_graph_1,
+//                     TransformerLevel::Default,
+//                     TransformerLevel::Level1,
+//                     /*opset_version*/ 13);
+// }
 
 TEST(TransposeOptimizerTests, TestAdd) {
   auto build_test_case_1 = [&](ModelTestBuilder& builder) {
@@ -4010,5 +4013,6 @@ TEST(TransposeOptimizerTests, RegressionTest_GitHubIssue10305) {
   ASSERT_STATUS_OK(session_object.Load(model_uri));
   ASSERT_STATUS_OK(session_object.Initialize());  // optimizers run during initialization
 }
+
 }  // namespace test
 }  // namespace onnxruntime
diff --git a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc
index ede3120efb..9174b0fc15 100644
--- a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc
+++ b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc
@@ -292,42 +292,7 @@ TEST(UpsampleOpTest, UpsampleOp4DBilinearTest) {
       7.0f, 7.5f, 8.0f, 8.5f, 9.0f, 9.0f, 9.0f, 9.0f};
 
   test.AddOutput<float>("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y);
-  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});  //TensorRT: results mismatch
-}
-
-TEST(UpsampleOpTest, UpsampleOp4DNhwcBilinearTest) {
-  OpTester test("Upsample");
-
-  std::vector<float> scales{1.0f, 2.0f, 4.0f, 1.0f};
-  test.AddAttribute("mode", "linear");
-  test.AddAttribute("scales", scales);
-
-  constexpr int64_t N = 2, H = 2, W = 3, C = 1;
-  std::vector<float> X = {1.0f, 2.0f, 3.0f,
-                          4.0f, 5.0f, 6.0f,
-
-                          7.0f, 8.0f, 9.0f,
-                          10.0f, 11.0f, 12.0f};
-
-  test.AddInput<float>("X", {N, H, W, C}, X);
-
-  std::vector<float> Y = {
-      1.0f, 1.25f, 1.5f, 1.75f, 2.0f, 2.25f, 2.5f, 2.75f, 3.0f, 3.0f, 3.0f, 3.0f,
-      2.5f, 2.75f, 3.0f, 3.25f, 3.5f, 3.75f, 4.0f, 4.25f, 4.5f, 4.5f, 4.5f, 4.5f,
-      4.0f, 4.25f, 4.5f, 4.75f, 5.0f, 5.25f, 5.5f, 5.75f, 6.0f, 6.0f, 6.0f, 6.0f,
-      4.0f, 4.25f, 4.5f, 4.75f, 5.0f, 5.25f, 5.5f, 5.75f, 6.0f, 6.0f, 6.0f, 6.0f,
-
-      7.0f, 7.25f, 7.5f, 7.75f, 8.0f, 8.25f, 8.5f, 8.75f, 9.0f, 9.0f, 9.0f, 9.0f,
-      8.5f, 8.75f, 9.0f, 9.25f, 9.5f, 9.75f, 10.0f, 10.25f, 10.5f, 10.5f, 10.5f, 10.5f,
-      10.0f, 10.25f, 10.5f, 10.75f, 11.0f, 11.25f, 11.5f, 11.75f, 12.0f, 12.0f, 12.0f, 12.0f,
-      10.0f, 10.25f, 10.5f, 10.75f, 11.0f, 11.25f, 11.5f, 11.75f, 12.0f, 12.0f, 12.0f, 12.0f};
-
-  test.AddOutput<float>("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y);
-  //CUDA: result mismatch due to not implementing NHWC support
-  //TensorRT: results mismatch
-  //ROCm: results mismatch
-  test.Run(OpTester::ExpectResult::kExpectSuccess, "",
-           {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
+  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: results mismatch
 }
 
 TEST(UpsampleOpTest, UpsampleOp2DBilinearTest) {
@@ -350,7 +315,7 @@ TEST(UpsampleOpTest, UpsampleOp2DBilinearTest) {
       3.0f, 3.5f, 4.0f, 4.5f, 5.0f, 5.0f, 5.0f, 5.0f};
 
   test.AddOutput<float>("Y", {(int64_t)(H * scales[0]), (int64_t)(W * scales[1])}, Y);
-  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});  //TensorRT: results mismatch
+  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: results mismatch
 }
 
 TEST(UpsampleOpTest, UpsampleOp4DBilinearTest_ScalesNoOp) {
diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
index 597cb6c290..fefd119f0c 100644
--- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
+++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
@@ -10,6 +10,7 @@
 #include "core/providers/tensorrt/tensorrt_provider_options.h"
 #include "core/providers/tensorrt/tensorrt_execution_provider_utils.h"
 #include <string>
+#include <thread>
 
 using namespace std;
 using namespace ONNX_NAMESPACE;
@@ -87,6 +88,190 @@ void CreateBaseModel(std::string model_name, std::string graph_name, std::vector
   status = onnxruntime::Model::Save(model, model_name);
 }
 
+void RunSession(InferenceSession& session_object,
+                RunOptions& run_options,
+                NameMLValMap& feeds,
+                std::vector<std::string> output_names,
+                std::vector<int64_t> expected_dims,
+                std::vector<float> expected_values) {
+    std::vector<OrtValue> fetches;
+    auto status = session_object.Run(run_options, feeds, output_names, &fetches);
+    ASSERT_TRUE(status.IsOK());
+    VerifyOutputs(fetches, expected_dims, expected_values);
+}
+
+void RunWithOneSessionSingleThreadInference(std::string model_name, std::string sess_log_id) {
+  SessionOptions so;
+  so.session_logid = sess_log_id;
+  RunOptions run_options;
+  run_options.run_tag = so.session_logid;
+  InferenceSession session_object{so, GetEnvironment()};
+  auto allocator_manager = session_object.GetAllocatorManager();
+  auto cuda_provider = DefaultCudaExecutionProvider();
+  cuda_provider->RegisterAllocator(allocator_manager);
+  auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU);
+  std::vector<int64_t> dims_mul_x = {1, 3, 2};
+  std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+  OrtValue ml_value_x;
+  CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x);
+  OrtValue ml_value_y;
+  CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y);
+  OrtValue ml_value_z;
+  CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z);
+  NameMLValMap feeds;
+  feeds.insert(std::make_pair("X", ml_value_x));
+  feeds.insert(std::make_pair("Y", ml_value_y));
+  feeds.insert(std::make_pair("Z", ml_value_z));
+
+  // prepare outputs
+  std::vector<std::string> output_names;
+  output_names.push_back("M");
+
+  // prepare expected inputs and outputs
+  std::vector<int64_t> expected_dims_mul_m = {1, 3, 2};
+  std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
+
+  OrtTensorRTProviderOptionsV2 params{
+      0,
+      0,
+      nullptr,
+      1000,
+      1,
+      1 << 30,
+      0,
+      0,
+      nullptr,
+      0,
+      0,
+      0,
+      0,
+      0,
+      nullptr,
+      0,
+      nullptr,
+      0};
+
+    params.trt_engine_cache_enable = 1;
+    std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
+    EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
+    auto status = session_object.Load(model_name);
+    ASSERT_TRUE(status.IsOK());
+    status = session_object.Initialize();
+    ASSERT_TRUE(status.IsOK());
+
+    // run inference
+    // TRT engine will be created and cached
+    // TRT profile will be created and cached only for dynamic input shape
+    // Data in profile,
+    // X: 1, 3, 3, 2, 2, 2
+    // Y: 1, 3, 3, 2, 2, 2
+    // Z: 1, 3, 3, 2, 2, 2
+    RunSession(session_object, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m);
+}
+
+void RunWithOneSessionMultiThreadsInference(std::string model_name, std::string sess_log_id) {
+  SessionOptions so;
+  so.session_logid = sess_log_id;
+  RunOptions run_options;
+  run_options.run_tag = so.session_logid;
+  InferenceSession session_object{so, GetEnvironment()};
+  auto allocator_manager = session_object.GetAllocatorManager();
+  auto cuda_provider = DefaultCudaExecutionProvider();
+  cuda_provider->RegisterAllocator(allocator_manager);
+  auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU);
+  std::vector<int64_t> dims_mul_x = {1, 3, 2};
+  std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+  OrtValue ml_value_x;
+  CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x);
+  OrtValue ml_value_y;
+  CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y);
+  OrtValue ml_value_z;
+  CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z);
+  NameMLValMap feeds;
+  feeds.insert(std::make_pair("X", ml_value_x));
+  feeds.insert(std::make_pair("Y", ml_value_y));
+  feeds.insert(std::make_pair("Z", ml_value_z));
+
+  // prepare outputs
+  std::vector<std::string> output_names;
+  output_names.push_back("M");
+
+  // prepare expected inputs and outputs
+  std::vector<int64_t> expected_dims_mul_m = {1, 3, 2};
+  std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
+
+  OrtTensorRTProviderOptionsV2 params{
+      0,
+      0,
+      nullptr,
+      1000,
+      1,
+      1 << 30,
+      0,
+      0,
+      nullptr,
+      0,
+      0,
+      0,
+      0,
+      0,
+      nullptr,
+      0,
+      nullptr,
+      0};
+
+    params.trt_engine_cache_enable = 1;
+    std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
+    EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
+    auto status = session_object.Load(model_name);
+    ASSERT_TRUE(status.IsOK());
+    status = session_object.Initialize();
+    ASSERT_TRUE(status.IsOK());
+
+    // run inference with multi-threads
+    // TRT engine will be created and cached
+    // TRT profile will be created and cached only for dynamic input shape
+    // Data in profile,
+    // X: 1, 3, 3, 2, 2, 2
+    // Y: 1, 3, 3, 2, 2, 2
+    // Z: 1, 3, 3, 2, 2, 2
+
+    std::vector<std::thread> threads;
+    int num_thread = 5;
+    for (int i = 0; i < num_thread; ++i)
+      threads.push_back(std::thread(RunSession, std::ref(session_object), std::ref(run_options), std::ref(feeds), std::ref(output_names), std::ref(expected_dims_mul_m), std::ref(expected_values_mul_m)));
+
+    for (auto& th : threads)
+      th.join();
+}
+
+TEST(TensorrtExecutionProviderTest, MultiThreadsTestWithOneSessionSingleThreadInference) {
+  std::vector<std::thread> threads;
+  std::string model_name = "trt_execution_provider_multithreading_test.onnx";
+  std::string graph_name = "multithreading_test";
+  std::string sess_log_id = "TRTEPMultiThreadingTestWithOneSessionSingleThread";
+  std::vector<int> dims = {1, 3, 2};
+  int num_thread = 5;
+
+  CreateBaseModel(model_name, graph_name, dims);
+
+  for (int i = 0; i < num_thread; ++i)
+    threads.push_back(std::thread(RunWithOneSessionSingleThreadInference, model_name, sess_log_id));
+
+  for (auto& th : threads)
+    th.join();
+}
+
+TEST(TensorrtExecutionProviderTest, MultiThreadsTestWithOneSessionMultiThreadsInference) {
+  std::string model_name = "trt_execution_provider_multithreading_test.onnx";
+  std::string graph_name = "multithreading_test";
+  std::string sess_log_id = "TRTEPMultiThreadingTestWithOneSessionMultiThreads";
+  std::vector<int> dims = {1, 3, 2};
+
+  CreateBaseModel(model_name, graph_name, dims);
+  RunWithOneSessionMultiThreadsInference(model_name, sess_log_id);
+}
+
 TEST_P(TensorrtExecutionProviderCacheTest, Run) {
   // GetParam() returns the parameter of following format:
   // ##cache type##_##input shape type##
diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc
index 00879dc4a3..899ea0a19e 100644
--- a/onnxruntime/test/shared_lib/test_inference.cc
+++ b/onnxruntime/test/shared_lib/test_inference.cc
@@ -168,7 +168,7 @@ static constexpr PATH_TYPE MATMUL_MODEL_URI = TSTR("testdata/matmul_1.onnx");
 #ifndef ORT_NO_RTTI
 static constexpr PATH_TYPE SEQUENCE_MODEL_URI = TSTR("testdata/sequence_length.onnx");
 #endif
-#ifdef USE_CUDA
+#if !defined(REDUCED_OPS_BUILD) && defined(USE_CUDA)
 static constexpr PATH_TYPE SEQUENCE_MODEL_URI_2 = TSTR("testdata/optional_sequence_tensor.onnx");
 #endif
 static constexpr PATH_TYPE CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_1.onnx");
@@ -2103,6 +2103,10 @@ TEST(CApiTest, GitHubIssue10179) {
   }
 }
 
+#endif
+
+// Reduced Ops build doesn't support If (16) yet
+#if !defined(REDUCED_OPS_BUILD) && defined(USE_CUDA)
 TEST(CApiTest, TestCudaMemcpyToHostWithSequenceTensors) {
   const auto* model_path = SEQUENCE_MODEL_URI_2;
   Ort::SessionOptions session_options{};
diff --git a/orttraining/orttraining/core/session/training_session.cc b/orttraining/orttraining/core/session/training_session.cc
index 30dd2a23f2..60b278ac7c 100644
--- a/orttraining/orttraining/core/session/training_session.cc
+++ b/orttraining/orttraining/core/session/training_session.cc
@@ -754,10 +754,13 @@ void TrainingSession::AddPreTrainingTransformers(const IExecutionProvider& execu
 }
 
 // Registers all the predefined transformers with transformer manager
-Status TrainingSession::AddPredefinedTransformers(GraphTransformerManager& transformer_manager,
-                                                  TransformerLevel graph_optimization_level,
-                                                  bool saving_runtime_optimizations) const {
-  ORT_RETURN_IF(saving_runtime_optimizations, "Saving runtime optimizations is not supported by TrainingSession.");
+Status TrainingSession::AddPredefinedTransformers(
+    GraphTransformerManager& transformer_manager,
+    TransformerLevel graph_optimization_level,
+    MinimalBuildOptimizationHandling minimal_build_optimization_handling) const {
+  ORT_RETURN_IF_NOT(
+      minimal_build_optimization_handling == MinimalBuildOptimizationHandling::ApplyFullBuildOptimizations,
+      "Only applying full build optimizations is supported by TrainingSession.");
 
   ORT_RETURN_IF_NOT(graph_optimization_level <= TransformerLevel::MaxLevel,
                     "Exceeded max transformer level. Current level is set to " +
diff --git a/orttraining/orttraining/core/session/training_session.h b/orttraining/orttraining/core/session/training_session.h
index 1150241dae..f6bf898583 100644
--- a/orttraining/orttraining/core/session/training_session.h
+++ b/orttraining/orttraining/core/session/training_session.h
@@ -485,9 +485,10 @@ class TrainingSession : public InferenceSession {
                                   TransformerLevel graph_optimization_level = TransformerLevel::MaxLevel);
 
   /** override the parent method in inference session for training specific transformers */
-  common::Status AddPredefinedTransformers(GraphTransformerManager& transformer_manager,
-                                           TransformerLevel graph_optimization_level,
-                                           bool saving_runtime_optimizations) const override;
+  common::Status AddPredefinedTransformers(
+      GraphTransformerManager& transformer_manager,
+      TransformerLevel graph_optimization_level,
+      MinimalBuildOptimizationHandling minimal_build_optimization_handling) const override;
 
   /** Perform auto-diff to add backward graph into the model.
   @param weights_to_train a set of weights to be training.
diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
index bf224e3af7..7d7896bb1e 100644
--- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
@@ -19,6 +19,13 @@ parameters:
   type: boolean
   default: false
 
+resources:
+  repositories:
+  - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
+    type: github
+    endpoint: ort-examples 
+    name: microsoft/onnxruntime-inference-examples 
+
 jobs:
 - template: templates/c-api-cpu.yml
   parameters:
@@ -269,9 +276,14 @@ jobs:
   - Linux_C_API_Packaging_GPU_TensorRT_x64
   condition: succeeded()
   steps:
-  - checkout: self
+  - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
+  - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
     submodules: false
+  - script: dir $(Build.SourcesDirectory)
   - template: templates/set-version-number-variables-step.yml
+    parameters:
+      versionFileDirectory: '$(Build.SourcesDirectory)/onnxruntime'
+      workingDirectory: '$(Build.SourcesDirectory)/onnxruntime'
   - task: DownloadPipelineArtifact@2
     displayName: 'Download Pipeline Artifact - Combined GPU'
     inputs:
@@ -287,7 +299,7 @@ jobs:
   - task: ShellScript@2
     displayName: 'Shell Script'
     inputs:
-      scriptPath: 'tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh'
+      scriptPath: 'onnxruntime/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh'
       args: '-a $(Build.BinariesDirectory)/tgz-artifacts'
       workingDirectory: '$(Build.BinariesDirectory)/tgz-artifacts'
 
@@ -305,10 +317,27 @@ jobs:
       PackageType: 'tarball'
       PackagePath: '$(Build.ArtifactStagingDirectory)'
       PackageName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
+      ScriptPath: '$(Build.SourcesDirectory)/onnxruntime/tools/nuget/validate_package.py'
       PlatformsSupported: 'linux-x64'
       VerifyNugetSigning: false
       workingDirectory: '$(Build.ArtifactStagingDirectory)'
 
+  - template: templates/get-docker-image-steps.yml
+    parameters:
+      ScriptName: onnxruntime/tools/ci_build/get_docker_image.py 
+      Dockerfile: onnxruntime/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_4_tensorrt8_2
+      Context: onnxruntime/tools/ci_build/github/linux/docker
+      DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
+      Repository: onnxruntimecuda114xtrt82build
+  - task: CmdLine@2
+    displayName: 'Test C API application for GPU package'
+    inputs:
+      script: |
+        docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/src_dir \
+        --volume $(Build.ArtifactStagingDirectory):/artifact_src -e NIGHTLY_BUILD onnxruntimecuda114xtrt82build \
+        /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet/run_capi_application.sh -o /src_dir/onnxruntime -p /artifact_src/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz -w /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet 
+      workingDirectory: '$(Build.ArtifactStagingDirectory)'
+
   - task: PublishPipelineArtifact@1
     inputs:
       targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
@@ -317,15 +346,26 @@ jobs:
 - job: Windows_Packaging_combined_GPU
   workspace:
     clean: all
-  pool: 'Win-CPU-2021'
+  pool: 'onnxruntime-gpu-tensorrt8-winbuild'
   dependsOn:
   - Windows_Packaging_gpu
   - Windows_Packaging_tensorrt
   condition: succeeded()
   steps:
-  - checkout: self
+  - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
+  - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
     submodules: false
+  - script: dir $(Build.SourcesDirectory)
+  - task: BatchScript@1
+    displayName: 'setup env'
+    inputs:
+        filename: '$(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\setup_env_gpu.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
   - template: templates/set-version-number-variables-step.yml
+    parameters:
+      versionFileDirectory: '$(Build.SourcesDirectory)\onnxruntime'
+      workingDirectory: '$(Build.SourcesDirectory)\onnxruntime'
   - task: DownloadPipelineArtifact@2
     displayName: 'Download Pipeline Artifact - Combined GPU'
     inputs:
@@ -342,7 +382,7 @@ jobs:
     displayName: 'PowerShell Script'
     inputs:
       targetType: filePath
-      filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_zip_files_gpu.ps1
+      filePath: $(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\extract_zip_files_gpu.ps1
 
   - script: |
      dir
@@ -352,7 +392,7 @@ jobs:
   - task: BatchScript@1
     displayName: 'Bundle CUDA/TRT EP binaries'
     inputs:
-      filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_dlls_gpu.bat
+      filename: $(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\bundle_dlls_gpu.bat
       workingFolder: $(Build.BinariesDirectory)\zip-artifacts
 
   - task: CopyFiles@2
@@ -367,10 +407,18 @@ jobs:
       PackageType: 'zip'
       PackagePath: '$(Build.ArtifactStagingDirectory)'
       PackageName: 'onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip'
+      ScriptPath: '$(Build.SourcesDirectory)\onnxruntime\tools\nuget\validate_package.py'
       PlatformsSupported: 'win-x64'
       VerifyNugetSigning: false
       workingDirectory: '$(Build.ArtifactStagingDirectory)'
 
+  - task: BatchScript@1
+    displayName: 'Test C API application for GPU package'
+    inputs:
+      filename: $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet\run_capi_application.bat
+      arguments: $(Build.SourcesDirectory)\onnxruntime $(Build.ArtifactStagingDirectory)\onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet 
+      workingFolder: '$(Build.ArtifactStagingDirectory)' 
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Pipeline Combined GPU Package Artifact'
     inputs:
diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml
index c32cb6e0d8..028e86d222 100644
--- a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml
@@ -86,7 +86,7 @@ jobs:
   # create and test mobile pods
   - script: |
       python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \
-        --build-dir "$(Build.BinariesDirectory)/ios_framework" \
+        --build-dir "$(Build.BinariesDirectory)/ios_framework_mobile" \
         --staging-dir "$(Build.BinariesDirectory)/staging" \
         --pod-version "${ORT_POD_VERSION}" \
         --test \
@@ -99,8 +99,8 @@ jobs:
   - script: |
       python tools/ci_build/github/apple/test_ios_packages.py \
         --fail_if_cocoapods_missing \
-        --framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \
-        --c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \
+        --framework_info_file "$(Build.BinariesDirectory)/ios_framework_mobile/framework_info.json" \
+        --c_framework_dir "$(Build.BinariesDirectory)/ios_framework_mobile/framework_out" \
         --variant Mobile \
         --test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_mobile" \
         --prepare_test_project_only
@@ -134,7 +134,7 @@ jobs:
   # create and test full pods
   - script: |
       python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \
-        --build-dir "$(Build.BinariesDirectory)/ios_framework" \
+        --build-dir "$(Build.BinariesDirectory)/ios_framework_full" \
         --staging-dir "$(Build.BinariesDirectory)/staging" \
         --pod-version "${ORT_POD_VERSION}" \
         --test \
@@ -146,8 +146,8 @@ jobs:
   - script: |
       python tools/ci_build/github/apple/test_ios_packages.py \
         --fail_if_cocoapods_missing \
-        --framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \
-        --c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \
+        --framework_info_file "$(Build.BinariesDirectory)/ios_framework_full/framework_info.json" \
+        --c_framework_dir "$(Build.BinariesDirectory)/ios_framework_full/framework_out" \
         --variant Full \
         --test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_full" \
         --prepare_test_project_only
diff --git a/tools/ci_build/github/azure-pipelines/mac-react-native-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-react-native-ci-pipeline.yml
index ce7b0d6496..4426e26693 100644
--- a/tools/ci_build/github/azure-pipelines/mac-react-native-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/mac-react-native-ci-pipeline.yml
@@ -10,292 +10,18 @@ parameters:
   default: 'nightly (@dev)'
 
 variables:
-  build_config: Release
-  pool_name: '$(PoolName)'
   ${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}:
-    npm_packaging_mode: 'dev'
+    NpmPackagingMode: 'dev'
   ${{ if eq(parameters.NpmPublish, 'release candidate (@rc)') }}:
-    npm_packaging_mode: 'rc'
+    NpmPackagingMode: 'rc'
   ${{ if eq(parameters.NpmPublish, 'production (@latest)') }}:
-    npm_packaging_mode: 'release'
+    NpmPackagingMode: 'release'
   ${{ if eq(parameters.NpmPublish, 'custom') }}:
-    npm_packaging_mode: '$(VersionSuffix)'
+    NpmPackagingMode: '$(VersionSuffix)'
 
 jobs:
-- template: templates/android-java-api-aar.yml
+- template: templates/react-native-ci.yml
   parameters:
-    buildConfig: '${{variables.build_config}}'
-    buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_mobile_aar_build_settings.json'
-    includedOpsConfig: '$(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config'
-    artifactName: 'onnxruntime-android-mobile-aar'
-    job_name_suffix: 'For_React_Native'
-    pool_name: '${{variables.pool_name}}'
-    packageName: 'onnxruntime-mobile'
-
-- job: ReactNative_CI
-  pool:
-    vmImage: 'macOS-11'
-  variables:
-    runCodesignValidationInjection: false
-  dependsOn:
-  - Android_Java_API_AAR_Packaging_For_React_Native
-  timeoutInMinutes: 120
-  steps:
-  # Onnx has no 3.9 python package available yet, need to use python 3.8 to avoid build onnx package
-  # pythonVersion can be updated in Azure pipeline settings
-  # https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=188
-  - task: UsePythonVersion@0
-    displayName: Use Python $(pythonVersion)
-    inputs:
-      versionSpec: $(pythonVersion)
-
-  - task: NodeTool@0
-    inputs:
-      versionSpec: '16.x'
-
-  - script:
-      brew install coreutils ninja npm yarn
-    displayName: Install coreutils, ninja, npm, and yarn
-
-  - script:
-      /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/android/setup_gradle_wrapper.sh $(pwd)
-    displayName: Setup gradle wrapper to use gradle 6.8.3
-
-  - script: |
-      python3 -m pip install -q flatbuffers
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: Install python modules
-
-  - script: |
-      python3 $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_ios_framework.py \
-        --config ${{variables.build_config}} \
-        --build_dir $(Build.BinariesDirectory)/ios_framework \
-        --include_ops_by_config $(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config \
-        $(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_mobile_ios_framework_build_settings.json
-      cd $(Build.BinariesDirectory)/ios_framework/framework_out
-      zip -r onnxruntime-mobile-c.zip .
-    displayName: Build iOS package
-
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      buildType: 'current'
-      artifactName: 'onnxruntime-android-mobile-aar'
-      targetPath: '$(Build.BinariesDirectory)/android-mobile-aar'
-    displayName: Download Android Aar artifacts
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.BinariesDirectory)/android-mobile-aar
-      contents: onnxruntime-mobile-*.aar
-      targetFolder: $(Build.SourcesDirectory)/js/react_native/android/libs
-    displayName: Copy Android package to React Native directory
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.BinariesDirectory)/ios_framework/framework_out
-      contents: onnxruntime-mobile-c.zip
-      targetFolder: $(Build.SourcesDirectory)/js/react_native/local_pods
-    displayName: Copy iOS package to React Native directory
-
-  - script: |
-      npm ci
-    workingDirectory: '$(Build.SourcesDirectory)/js'
-    displayName: npm ci js
-
-  - script: |
-      npm ci
-    workingDirectory: '$(Build.SourcesDirectory)/js/common'
-    displayName: npm ci js/common
-
-  - script: |
-      yarn
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native'
-    displayName: yarn js/react_native
-
-  - script: |
-      python3 tools/python/run_android_emulator.py \
-        --android-sdk-root $(ANDROID_SDK_ROOT) \
-        --create-avd --system-image "system-images;android-30;google_apis;x86_64" \
-        --start --emulator-extra-args="-partition-size 4096" \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: Start Android Emulator
-
-  - script: |
-      xcrun simctl create iPhoneRNTest com.apple.CoreSimulator.SimDeviceType.iPhone-13
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios'
-    displayName: Start iOS Simulator
-
-  - task: Gradle@3
-    inputs:
-      gradleWrapperFile: '$(Build.SourcesDirectory)/js/react_native/android/gradlew'
-      workingDirectory: '$(Build.SourcesDirectory)/js/react_native/android'
-      options: '--stacktrace'
-      tasks: 'connectedDebugAndroidTest'
-      publishJUnitResults: true
-      testResultsFiles: '**/TEST-*.xml'
-      testRunTitle: 'React Native Android Instrumented Test results'
-      javaHomeOption: 'JDKVersion'
-      sonarQubeRunAnalysis: false
-      spotBugsAnalysis: false
-    displayName: Run React Native Android Instrumented Tests
-    continueOnError: false
-
-  - script: |
-      pod install
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios'
-    displayName: Pod install for onnxruntime react native ios bridge library
-
-  - task: Xcode@5
-    inputs:
-      actions: 'test'
-      configuration: 'Debug'
-      sdk: 'iphonesimulator'
-      xcWorkspacePath: '$(Build.SourcesDirectory)/js/react_native/ios/OnnxruntimeModule.xcworkspace'
-      scheme: 'OnnxruntimeModuleTest'
-      packageApp: false
-      destinationPlatformOption: 'iOS'
-      destinationSimulators: 'iPhone 13,OS=latest'
-      workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios'
-      xcprettyArgs: '--output build/reports/test-results.xml'
-      publishJUnitResults: true
-      testRunTitle: 'React Native iOS Instrumented Test Results'
-    displayName: Run React Native iOS Instrumented Tests
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/ios/build/reports/test-results.xml'
-      failTaskOnFailedTests: true
-      testRunTitle: 'React Native iOS Instrumented Test results'
-    condition: succeededOrFailed()
-    displayName: Publish React Native iOS Instrumented Test Results
-
-  - script: |
-      yarn prepack-e2e
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native'
-    displayName: Prepare Android and iOS e2e tests
-
-  - task: PowerShell@2
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1'
-      arguments: '"-dev.$(Get-Date -Format yyyyMMdd)-$(git rev-parse --short HEAD)" $(Build.SourcesDirectory) react_native'
-      workingDirectory: '$(Build.SourcesDirectory)'
-      errorActionPreference: stop
-    displayName: Pack NPM packages
-
-  - script: |
-      mv $(Build.SourcesDirectory)/js/common/onnxruntime-common*.tgz onnxruntime-common.tgz
-      yarn add --no-lockfile file:./onnxruntime-common.tgz
-      mv $(Build.SourcesDirectory)/js/react_native/onnxruntime-react-native*.tgz onnxruntime-react-native.tgz
-      yarn add --no-lockfile file:./onnxruntime-react-native.tgz
-      yarn
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e'
-    displayName: Bootstrap Android and iOS e2e tests
-
-  - script: |
-      pod install
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios'
-    displayName: Pod install for onnxruntime react native ios e2e tests
-
-  - script: |
-      keytool -genkey -v -keystore debug.keystore -alias androiddebugkey -storepass android \
-        -keypass android -keyalg RSA -keysize 2048 -validity 999999 -dname "CN=Android Debug,O=Android,C=US"
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/android'
-    displayName: Generate a debug keystore
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.BinariesDirectory)/android-mobile-aar
-      contents: onnxruntime-mobile-*.aar
-      targetFolder: $(Build.SourcesDirectory)/js/react_native/e2e/node_modules/onnxruntime-react-native/android/libs
-    displayName: Copy Android package to React Native e2e directory
-
-  - task: Gradle@3
-    inputs:
-      gradleWrapperFile: '$(Build.SourcesDirectory)/js/react_native/e2e/android/gradlew'
-      workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/android'
-      options: '--stacktrace'
-      tasks: ':app:connectedDebugAndroidTest'
-      publishJUnitResults: true
-      testResultsFiles: '**/TEST-*.xml'
-      testRunTitle: 'React Native Android e2e Test results'
-      javaHomeOption: 'JDKVersion'
-      sonarQubeRunAnalysis: false
-      spotBugsAnalysis: false
-    displayName: Run React Native Android e2e Tests
-    continueOnError: false
-
-  - script: |
-      export FORCE_BUNDLING=1
-      export RCT_NO_LAUNCH_PACKAGER=1
-      export ENTRY_FILE=index.tsx
-      xcrun xcodebuild test -workspace $(Build.SourcesDirectory)/js/react_native/e2e/ios/OnnxruntimeModuleExample.xcworkspace \
-        -scheme OnnxruntimeModuleExample -destination 'platform=iOS Simulator,OS=latest,name=iPhoneRNTest' \
-        -derivedDataPath $(Build.BinariesDirectory)/react_native/ios_e2e_test/derived_data | xcpretty -r junit --no-color \
-        --output $(Build.SourcesDirectory)/js/react_native/e2e/ios/build/reports/test-results.xml
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e'
-    displayName: Run React Native iOS e2e tests
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/e2e/ios/build/reports/test-results.xml'
-      failTaskOnFailedTests: true
-      testRunTitle: 'React Native iOS e2e Test results'
-    condition: succeededOrFailed()
-    displayName: Publish React Native iOS e2e Test Results
-
-  - script: |
-      python3 tools/python/run_android_emulator.py \
-        --android-sdk-root $(ANDROID_SDK_ROOT) \
-        --stop \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: Stop Android Emulator
-    condition: always()
-
-  - script: |
-      xcrun simctl delete iPhoneRNTest
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios'
-    displayName: Stop iOS Simulator
-    condition: always()
-
-  - script: |
-      git restore .
-      cd react_native
-      yarn prepack-rel
-    workingDirectory: '$(Build.SourcesDirectory)/js'
-    displayName: Restore git changes and prepack for npm publish
-
-  - task: PowerShell@2
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1'
-      arguments: '"${{variables.npm_packaging_mode}}" $(Build.SourcesDirectory) react_native'
-      workingDirectory: '$(Build.SourcesDirectory)'
-      errorActionPreference: stop
-    displayName: Pack NPM packages
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)/js/common
-      contents: onnxruntime-common*.tgz
-      targetFolder: $(Build.ArtifactStagingDirectory)
-    displayName: 'Create Artifacts onnxruntime-common'
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)/js/react_native
-      contents: onnxruntime-react-native*.tgz
-      targetFolder: $(Build.ArtifactStagingDirectory)
-    displayName: Create Artifacts onnxruntime-react-native
-
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: 'NPM_packages'
-      targetPath: '$(Build.ArtifactStagingDirectory)'
-    displayName: Publish Pipeline Artifact
-
-  - template: templates/component-governance-component-detection-steps.yml
-    parameters :
-      condition : 'succeeded'
-
-  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
-    displayName: Clean Agent Directories
-    condition: always()
+    NpmPackagingMode: ${{ variables.NpmPackagingMode }}
+    BuildConfig: 'Release'
+    PoolName: 'Linux-CPU-2019'
diff --git a/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml
new file mode 100644
index 0000000000..b4dd4f7a79
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/npm-packaging-pipeline.yml
@@ -0,0 +1,98 @@
+parameters:
+- name: NpmPublish
+  displayName: 'NPM packages publish configuration'
+  type: string
+  values:
+  - 'nightly (@dev)'
+  - 'release candidate (@rc)'
+  - 'production (@latest)'
+  - 'custom'
+  default: 'nightly (@dev)'
+
+- name: NodePipelineId
+  displayName: 'Node npm package build Id'
+  type: string
+  default: 'latest'
+
+variables:
+  # pipeline should define the following varaibles
+  #   ExtraBuildArgs
+  #   VersionSuffix
+
+  ${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}:
+    NpmPackagingMode: 'dev'
+  ${{ if eq(parameters.NpmPublish, 'release candidate (@rc)') }}:
+    NpmPackagingMode: 'rc'
+  ${{ if eq(parameters.NpmPublish, 'production (@latest)') }}:
+    NpmPackagingMode: 'release'
+  ${{ if eq(parameters.NpmPublish, 'custom') }}:
+    NpmPackagingMode: '$(VersionSuffix)'
+
+stages:
+- template: templates/web-ci.yml
+  parameters:
+    NpmPackagingMode: ${{ variables.NpmPackagingMode }}
+    IsReleasePipeline: true
+    PoolName: 'Win-CPU-2021'
+    PackageName: 'onnxruntime-web'
+
+- stage: Build_React_Native
+  dependsOn: Extract_commit
+  jobs:
+  - template: templates/react-native-ci.yml
+    parameters:
+      NpmPackagingMode: ${{ variables.NpmPackagingMode }}
+      BuildConfig: 'Release'
+      PoolName: 'Linux-CPU'
+      PackageName: 'onnxruntime-react-native'
+
+- stage: Download_Node_Package
+  dependsOn:
+  - Build_React_Native
+  - Build_web_Release
+  - Build_web_Debug
+  jobs:
+  - job: Download_Node_Package
+    pool: 'Win-CPU-2021'
+    variables:
+      runCodesignValidationInjection: false
+    timeoutInMinutes: 10
+    steps:
+
+    - ${{ if eq(parameters.NodePipelineId, 'latest') }}:
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          buildType: 'specific'
+          project: '530acbc4-21bc-487d-8cd8-348ff451d2ff'
+          definition: '940'
+          specificBuildWithTriggering: true
+          buildVersionToDownload: 'latest'
+          artifactName: 'NPM_packages'
+          targetPath: '$(Pipeline.Workspace)'
+        displayName: 'Download onnxruntime-node Pipeline Artifact'
+      
+    - ${{ if ne(parameters.NodePipelineId, 'latest') }}:
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          buildType: 'specific'
+          project: '530acbc4-21bc-487d-8cd8-348ff451d2ff'
+          definition: '940'
+          buildVersionToDownload: 'specific'
+          pipelineId: '${{ parameters.NodePipelineId }}'
+          artifactName: 'NPM_packages'
+          targetPath: '$(Pipeline.Workspace)'
+        displayName: 'Download onnxruntime-node Pipeline Artifact'
+    
+    - task: CopyFiles@2
+      inputs:
+        sourceFolder: $(Pipeline.Workspace)
+        contents: onnxruntime-*.tgz
+        targetFolder: $(Build.ArtifactStagingDirectory)
+      displayName: 'Copy onnxruntime-node Artifacts'
+    
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'onnxruntime-node'
+        targetPath: '$(Build.ArtifactStagingDirectory)'
+      displayName: 'Publish onnxruntime-node Pipeline Artifact'
+      
\ No newline at end of file
diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
index 62c133be6b..bad19f6e41 100644
--- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
@@ -13,6 +13,9 @@ parameters:
 - name: UseImageCacheContainerRegistry
   type: boolean
   default: true
+- name: ScriptName
+  type: string
+  default: "tools/ci_build/get_docker_image.py" 
 
 steps:
 - ${{ if eq(parameters.UseImageCacheContainerRegistry, true) }}:
@@ -20,7 +23,7 @@ steps:
     parameters:
       Steps:
       - script: |
-          tools/ci_build/get_docker_image.py \
+          ${{ parameters.ScriptName }} \
             --dockerfile "${{ parameters.Dockerfile }}" \
             --context "${{ parameters.Context }}" \
             --docker-build-args "${{ parameters.DockerBuildArgs }}" \
@@ -30,7 +33,7 @@ steps:
       ContainerRegistry: onnxruntimebuildcache
 - ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}:
   - script: |
-      tools/ci_build/get_docker_image.py \
+      ${{ parameters.ScriptName }} \
         --dockerfile "${{ parameters.Dockerfile }}" \
         --context "${{ parameters.Context }}" \
         --docker-build-args "${{ parameters.DockerBuildArgs }}" \
diff --git a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml
new file mode 100644
index 0000000000..38e239a3fe
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml
@@ -0,0 +1,300 @@
+parameters:
+- name: NpmPackagingMode
+  displayName: 'NPM packages publish configuration'
+  type: string
+  default: 'dev'
+- name: BuildConfig
+  displayName: 'Build config'
+  type: string
+  values:
+  - 'Release'
+  - 'MinSizeRel'
+  - 'Debug'
+  - 'RelWithDebugInfo'
+  default: 'Release'
+- name: PoolName
+  displayName: 'Pool name'
+  type: string
+- name: PackageName
+  displayName: 'Package name'
+  type: string
+  default: 'NPM_packages'
+
+jobs:
+- template: android-java-api-aar.yml
+  parameters:
+    buildConfig: '${{parameters.BuildConfig}}'
+    buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_mobile_aar_build_settings.json'
+    includedOpsConfig: '$(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config'
+    artifactName: 'onnxruntime-android-mobile-aar'
+    job_name_suffix: 'For_React_Native'
+    pool_name: '${{parameters.PoolName}}'
+    packageName: 'onnxruntime-mobile'
+
+- job: ReactNative_CI
+  pool:
+    vmImage: 'macOS-11'
+  variables:
+    runCodesignValidationInjection: false
+  dependsOn:
+  - Android_Java_API_AAR_Packaging_For_React_Native
+  timeoutInMinutes: 120
+  steps:
+  # Onnx has no 3.9 python package available yet, need to use python 3.8 to avoid build onnx package
+  # pythonVersion can be updated in Azure pipeline settings
+  # https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=188
+  - task: UsePythonVersion@0
+    displayName: Use Python $(pythonVersion)
+    inputs:
+      versionSpec: $(pythonVersion)
+
+  - task: NodeTool@0
+    inputs:
+      versionSpec: '16.x'
+
+  - script:
+      brew install coreutils ninja npm yarn
+    displayName: Install coreutils, ninja, npm, and yarn
+
+  - script:
+      /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/android/setup_gradle_wrapper.sh $(pwd)
+    displayName: Setup gradle wrapper to use gradle 6.8.3
+
+  - script: |
+      python3 -m pip install -q flatbuffers
+    workingDirectory: '$(Build.BinariesDirectory)'
+    displayName: Install python modules
+
+  - script: |
+      python3 $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_ios_framework.py \
+        --config ${{parameters.BuildConfig}} \
+        --build_dir $(Build.BinariesDirectory)/ios_framework \
+        --include_ops_by_config $(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config \
+        $(Build.SourcesDirectory)/tools/ci_build/github/js/react_native_e2e_mobile_ios_framework_build_settings.json
+      cd $(Build.BinariesDirectory)/ios_framework/framework_out
+      zip -r onnxruntime-mobile-c.zip .
+    displayName: Build iOS package
+
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      buildType: 'current'
+      artifactName: 'onnxruntime-android-mobile-aar'
+      targetPath: '$(Build.BinariesDirectory)/android-mobile-aar'
+    displayName: Download Android Aar artifacts
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)/android-mobile-aar
+      contents: onnxruntime-mobile-*.aar
+      targetFolder: $(Build.SourcesDirectory)/js/react_native/android/libs
+    displayName: Copy Android package to React Native directory
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)/ios_framework/framework_out
+      contents: onnxruntime-mobile-c.zip
+      targetFolder: $(Build.SourcesDirectory)/js/react_native/local_pods
+    displayName: Copy iOS package to React Native directory
+
+  - script: |
+      npm ci
+    workingDirectory: '$(Build.SourcesDirectory)/js'
+    displayName: npm ci js
+
+  - script: |
+      npm ci
+    workingDirectory: '$(Build.SourcesDirectory)/js/common'
+    displayName: npm ci js/common
+
+  - script: |
+      yarn
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native'
+    displayName: yarn js/react_native
+
+  - script: |
+      python3 tools/python/run_android_emulator.py \
+        --android-sdk-root $(ANDROID_SDK_ROOT) \
+        --create-avd --system-image "system-images;android-30;google_apis;x86_64" \
+        --start --emulator-extra-args="-partition-size 4096" \
+        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
+    displayName: Start Android Emulator
+
+  - script: |
+      xcrun simctl create iPhoneRNTest com.apple.CoreSimulator.SimDeviceType.iPhone-13
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios'
+    displayName: Start iOS Simulator
+
+  - task: Gradle@3
+    inputs:
+      gradleWrapperFile: '$(Build.SourcesDirectory)/js/react_native/android/gradlew'
+      workingDirectory: '$(Build.SourcesDirectory)/js/react_native/android'
+      options: '--stacktrace'
+      tasks: 'connectedDebugAndroidTest'
+      publishJUnitResults: true
+      testResultsFiles: '**/TEST-*.xml'
+      testRunTitle: 'React Native Android Instrumented Test results'
+      javaHomeOption: 'JDKVersion'
+      sonarQubeRunAnalysis: false
+      spotBugsAnalysis: false
+    displayName: Run React Native Android Instrumented Tests
+    continueOnError: false
+
+  - script: |
+      pod install
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios'
+    displayName: Pod install for onnxruntime react native ios bridge library
+
+  - task: Xcode@5
+    inputs:
+      actions: 'test'
+      configuration: 'Debug'
+      sdk: 'iphonesimulator'
+      xcWorkspacePath: '$(Build.SourcesDirectory)/js/react_native/ios/OnnxruntimeModule.xcworkspace'
+      scheme: 'OnnxruntimeModuleTest'
+      packageApp: false
+      destinationPlatformOption: 'iOS'
+      destinationSimulators: 'iPhone 13,OS=latest'
+      workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios'
+      xcprettyArgs: '--output build/reports/test-results.xml'
+      publishJUnitResults: true
+      testRunTitle: 'React Native iOS Instrumented Test Results'
+    displayName: Run React Native iOS Instrumented Tests
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/ios/build/reports/test-results.xml'
+      failTaskOnFailedTests: true
+      testRunTitle: 'React Native iOS Instrumented Test results'
+    condition: succeededOrFailed()
+    displayName: Publish React Native iOS Instrumented Test Results
+
+  - script: |
+      yarn prepack-e2e
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native'
+    displayName: Prepare Android and iOS e2e tests
+
+  - task: PowerShell@2
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1'
+      arguments: '"-dev.$(Get-Date -Format yyyyMMdd)-$(git rev-parse --short HEAD)" $(Build.SourcesDirectory) react_native'
+      workingDirectory: '$(Build.SourcesDirectory)'
+      errorActionPreference: stop
+    displayName: Pack NPM packages
+
+  - script: |
+      mv $(Build.SourcesDirectory)/js/common/onnxruntime-common*.tgz onnxruntime-common.tgz
+      yarn add --no-lockfile file:./onnxruntime-common.tgz
+      mv $(Build.SourcesDirectory)/js/react_native/onnxruntime-react-native*.tgz onnxruntime-react-native.tgz
+      yarn add --no-lockfile file:./onnxruntime-react-native.tgz
+      yarn
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e'
+    displayName: Bootstrap Android and iOS e2e tests
+
+  - script: |
+      pod install
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios'
+    displayName: Pod install for onnxruntime react native ios e2e tests
+
+  - script: |
+      keytool -genkey -v -keystore debug.keystore -alias androiddebugkey -storepass android \
+        -keypass android -keyalg RSA -keysize 2048 -validity 999999 -dname "CN=Android Debug,O=Android,C=US"
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/android'
+    displayName: Generate a debug keystore
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)/android-mobile-aar
+      contents: onnxruntime-mobile-*.aar
+      targetFolder: $(Build.SourcesDirectory)/js/react_native/e2e/node_modules/onnxruntime-react-native/android/libs
+    displayName: Copy Android package to React Native e2e directory
+
+  - task: Gradle@3
+    inputs:
+      gradleWrapperFile: '$(Build.SourcesDirectory)/js/react_native/e2e/android/gradlew'
+      workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/android'
+      options: '--stacktrace'
+      tasks: ':app:connectedDebugAndroidTest'
+      publishJUnitResults: true
+      testResultsFiles: '**/TEST-*.xml'
+      testRunTitle: 'React Native Android e2e Test results'
+      javaHomeOption: 'JDKVersion'
+      sonarQubeRunAnalysis: false
+      spotBugsAnalysis: false
+    displayName: Run React Native Android e2e Tests
+    continueOnError: false
+
+  - script: |
+      export FORCE_BUNDLING=1
+      export RCT_NO_LAUNCH_PACKAGER=1
+      export ENTRY_FILE=index.tsx
+      xcrun xcodebuild test -workspace $(Build.SourcesDirectory)/js/react_native/e2e/ios/OnnxruntimeModuleExample.xcworkspace \
+        -scheme OnnxruntimeModuleExample -destination 'platform=iOS Simulator,OS=latest,name=iPhoneRNTest' \
+        -derivedDataPath $(Build.BinariesDirectory)/react_native/ios_e2e_test/derived_data | xcpretty -r junit --no-color \
+        --output $(Build.SourcesDirectory)/js/react_native/e2e/ios/build/reports/test-results.xml
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e'
+    displayName: Run React Native iOS e2e tests
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/e2e/ios/build/reports/test-results.xml'
+      failTaskOnFailedTests: true
+      testRunTitle: 'React Native iOS e2e Test results'
+    condition: succeededOrFailed()
+    displayName: Publish React Native iOS e2e Test Results
+
+  - script: |
+      python3 tools/python/run_android_emulator.py \
+        --android-sdk-root $(ANDROID_SDK_ROOT) \
+        --stop \
+        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
+    displayName: Stop Android Emulator
+    condition: always()
+
+  - script: |
+      xcrun simctl delete iPhoneRNTest
+    workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios'
+    displayName: Stop iOS Simulator
+    condition: always()
+
+  - script: |
+      git restore .
+      cd react_native
+      yarn prepack-rel
+    workingDirectory: '$(Build.SourcesDirectory)/js'
+    displayName: Restore git changes and prepack for npm publish
+
+  - task: PowerShell@2
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1'
+      arguments: '"${{parameters.NpmPackagingMode}}" $(Build.SourcesDirectory) react_native'
+      workingDirectory: '$(Build.SourcesDirectory)'
+      errorActionPreference: stop
+    displayName: Pack NPM packages
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.SourcesDirectory)/js/common
+      contents: onnxruntime-common*.tgz
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName: 'Create Artifacts onnxruntime-common'
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: $(Build.SourcesDirectory)/js/react_native
+      contents: onnxruntime-react-native*.tgz
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    displayName: Create Artifacts onnxruntime-react-native
+
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: '${{parameters.PackageName}}'
+      targetPath: '$(Build.ArtifactStagingDirectory)'
+    displayName: Publish Pipeline Artifact
+
+  - template: component-governance-component-detection-steps.yml
+    parameters :
+      condition : 'succeeded'
+
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: Clean Agent Directories
+    condition: always()
diff --git a/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml
index dcbdcffb73..ba2f2bfca7 100644
--- a/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml
@@ -1,4 +1,8 @@
 # Sets version number from VERSION.txt into a variable. As well as the git commit hash.
+parameters:
+  versionFileDirectory: "$(Build.SourcesDirectory)"
+  workingDirectory: "$(Build.SourcesDirectory)"
+
 
 steps:
 - task: CmdLine@2
@@ -6,7 +10,7 @@ steps:
   inputs:
     script: |
         SETLOCAL EnableDelayedExpansion
-        set /p _OnnxRuntimeVersion=<$(Build.SourcesDirectory)\VERSION_NUMBER 
+        set /p _OnnxRuntimeVersion=<${{parameters.versionFileDirectory}}\VERSION_NUMBER 
         @echo ##vso[task.setvariable variable=OnnxRuntimeVersion;]%_OnnxRuntimeVersion%
         
         FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse HEAD`) DO (
@@ -17,14 +21,14 @@ steps:
             @echo ##vso[task.setvariable variable=OnnxRuntimeGitCommitHashShort;]%%F
         )
         
-    workingDirectory: '$(Build.SourcesDirectory)'
+    workingDirectory: ${{parameters.workingDirectory}} 
   condition: eq(variables['Agent.OS'], 'Windows_NT') 
 
 - task: CmdLine@2
   displayName: 'Set version number variables for Unix'
   inputs:
     script: |
-        _OnnxRuntimeVersion=$(head -1 $(Build.SourcesDirectory)/VERSION_NUMBER) 
+        _OnnxRuntimeVersion=$(head -1 ${{parameters.versionFileDirectory}}/VERSION_NUMBER) 
         echo "##vso[task.setvariable variable=OnnxRuntimeVersion;]$_OnnxRuntimeVersion"
         
         _OnnxRuntimeGitCommitHash=$(git rev-parse HEAD)
@@ -33,5 +37,5 @@ steps:
         _OnnxRuntimeGitCommitHash=$(git rev-parse --short=8 HEAD)
         echo "##vso[task.setvariable variable=OnnxRuntimeGitCommitHashShort;]$_OnnxRuntimeGitCommitHash"
 
-    workingDirectory: '$(Build.SourcesDirectory)'
-  condition: not(eq(variables['Agent.OS'], 'Windows_NT'))
\ No newline at end of file
+    workingDirectory: ${{parameters.workingDirectory}} 
+  condition: not(eq(variables['Agent.OS'], 'Windows_NT'))
diff --git a/tools/ci_build/github/azure-pipelines/templates/validate-package.yml b/tools/ci_build/github/azure-pipelines/templates/validate-package.yml
index 0d9dde17c3..41707e0e99 100644
--- a/tools/ci_build/github/azure-pipelines/templates/validate-package.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/validate-package.yml
@@ -4,6 +4,7 @@ parameters:
   PackageType: ''
   PackageName: ''
   PackagePath: ''
+  ScriptPath: '$(Build.SourcesDirectory)/tools/nuget/validate_package.py'
   workingDirectory: "$(Build.BinariesDirectory)"
 
 steps:
@@ -15,6 +16,6 @@ steps:
     - task: PythonScript@0
       displayName: 'Validate Package'
       inputs:
-        scriptPath: '$(Build.SourcesDirectory)/tools/nuget/validate_package.py'
+        scriptPath: '${{parameters.ScriptPath}}'
         arguments: '--package_type ${{parameters.PackageType}} --package_name ${{parameters.PackageName}} --package_path ${{parameters.PackagePath}} --platforms_supported ${{parameters.PlatformsSupported}} --verify_nuget_signing ${{parameters.VerifyNugetSigning}}'
         workingDirectory: ${{parameters.workingDirectory}} 
diff --git a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml
index d390b1b6a4..cba20697c6 100644
--- a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml
@@ -11,6 +11,10 @@ parameters:
   displayName: 'Agent pool name'
   type: string
   default: 'Win-CPU-2019'
+- name: PackageName
+  displayName: 'Package name'
+  type: string
+  default: 'NPM_packages'
 
 stages:
 - stage: Extract_commit
@@ -54,6 +58,7 @@ stages:
       BuildConfig: 'Debug'
       NpmPackagingMode: ${{ parameters.NpmPackagingMode }}
       PoolName: ${{ parameters.PoolName }}
+      PackageName: ${{ parameters.PackageName }}
 
 - stage: Build_wasm_Release
   dependsOn: Extract_commit
@@ -74,6 +79,7 @@ stages:
       BuildConfig: 'Release'
       NpmPackagingMode: ${{ parameters.NpmPackagingMode }}
       PoolName: ${{ parameters.PoolName }}
+      PackageName: ${{ parameters.PackageName }}
 
 - ${{ if ne(parameters.IsReleasePipeline, 'true') }}:
   - stage: Test_web_BrowserStack
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml
index 5bcfa2e06a..c90899714a 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml
@@ -15,6 +15,11 @@ parameters:
   type: string
   default: 'Win-CPU-2019'
 
+- name: PackageName
+  displayName: 'Package name'
+  type: string
+  default: 'NPM_packages'
+
 jobs:
 - job: build_onnxruntime_web
   pool: ${{ parameters.PoolName }}
@@ -158,7 +163,7 @@ jobs:
     condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release'))
   - task: PublishPipelineArtifact@0
     inputs:
-      artifactName: 'NPM_packages'
+      artifactName: '${{ parameters.PackageName }}'
       targetPath: '$(Build.ArtifactStagingDirectory)'
     displayName: 'Publish Pipeline Artifact'
     condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release'))
diff --git a/tools/ci_build/github/windows/setup_env_gpu.bat b/tools/ci_build/github/windows/setup_env_gpu.bat
index ce216eb587..1e8bb45181 100644
--- a/tools/ci_build/github/windows/setup_env_gpu.bat
+++ b/tools/ci_build/github/windows/setup_env_gpu.bat
@@ -1,2 +1,2 @@
-set PATH=C:\azcopy;C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;%PATH%
+set PATH=C:\azcopy;C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH%
 set GRADLE_OPTS=-Dorg.gradle.daemon=false
diff --git a/tools/python/create_reduced_build_config.py b/tools/python/create_reduced_build_config.py
index 26caf694ef..9062c97765 100644
--- a/tools/python/create_reduced_build_config.py
+++ b/tools/python/create_reduced_build_config.py
@@ -3,10 +3,18 @@
 # Licensed under the MIT License.
 
 import argparse
-import os
 import onnx
 import pathlib
 import sys
+import typing
+
+from util.file_utils import files_from_file_or_dir, path_match_suffix_ignore_case
+
+
+def _get_suffix_match_predicate(suffix: str):
+    def predicate(file_path: pathlib.Path):
+        return path_match_suffix_ignore_case(file_path, suffix)
+    return predicate
 
 
 def _extract_ops_from_onnx_graph(graph, operators, domain_opset_map):
@@ -51,39 +59,29 @@ def _process_onnx_model(model_path, required_ops):
         _extract_ops_from_onnx_graph(model.graph, required_ops, domain_opset_map)
 
 
-def _extract_ops_from_onnx_model(model_path_or_dir):
-    '''Extract ops from a single ONNX model, or all ONNX models found by recursing model_path_or_dir'''
-
-    if not os.path.exists(model_path_or_dir):
-        raise ValueError('Path to model/s does not exist: {}'.format(model_path_or_dir))
+def _extract_ops_from_onnx_model(model_files: typing.Iterable[pathlib.Path]):
+    '''Extract ops from ONNX models'''
 
     required_ops = {}
 
-    if os.path.isfile(model_path_or_dir):
-        _process_onnx_model(model_path_or_dir, required_ops)
-    else:
-        for root, _, files in os.walk(model_path_or_dir):
-            for file in files:
-                if file.lower().endswith('.onnx'):
-                    model_path = os.path.join(root, file)
-                    _process_onnx_model(model_path, required_ops)
+    for model_file in model_files:
+        if not model_file.is_file():
+            raise ValueError(f"Path is not a file: '{model_file}'")
+        _process_onnx_model(model_file, required_ops)
 
     return required_ops
 
 
-def create_config_from_onnx_models(model_path_or_dir: str, output_file: str):
+def create_config_from_onnx_models(model_files: typing.Iterable[pathlib.Path], output_file: pathlib.Path):
 
-    required_ops = _extract_ops_from_onnx_model(model_path_or_dir)
+    required_ops = _extract_ops_from_onnx_model(model_files)
 
-    directory, filename = os.path.split(output_file)
-    if not filename:
-        raise RuntimeError("Invalid output path for configuation: {}".format(output_file))
-
-    if not os.path.exists(directory):
-        os.makedirs(directory)
+    output_file.parent.mkdir(parents=True, exist_ok=True)
 
     with open(output_file, 'w') as out:
-        out.write("# Generated from ONNX models path of {}\n".format(model_path_or_dir))
+        out.write("# Generated from ONNX model/s:\n")
+        for model_file in sorted(model_files):
+            out.write(f"# - {model_file}\n")
 
         for domain in sorted(required_ops.keys()):
             for opset in sorted(required_ops[domain].keys()):
@@ -129,10 +127,13 @@ def main():
         config_path = config_path.joinpath(filename)
 
     if args.format == 'ONNX':
-        create_config_from_onnx_models(model_path_or_dir, config_path)
+        model_files = files_from_file_or_dir(model_path_or_dir, _get_suffix_match_predicate(".onnx"))
+        create_config_from_onnx_models(model_files, config_path)
     else:
         from util.ort_format_model import create_config_from_models as create_config_from_ort_models
-        create_config_from_ort_models(model_path_or_dir, config_path, args.enable_type_reduction)
+
+        model_files = files_from_file_or_dir(model_path_or_dir, _get_suffix_match_predicate(".ort"))
+        create_config_from_ort_models(model_files, config_path, args.enable_type_reduction)
 
         # Debug code to validate that the config parsing matches
         # from util import parse_config
diff --git a/tools/python/util/__init__.py b/tools/python/util/__init__.py
index 759871c22f..5dcc1bdd9f 100644
--- a/tools/python/util/__init__.py
+++ b/tools/python/util/__init__.py
@@ -4,8 +4,6 @@
 from .get_azcopy import get_azcopy
 from .logger import get_logger
 from .platform_helpers import (is_windows, is_macOS, is_linux)
-# Test what is needed here to use in a script
-# from .pytorch_export_helpers import infer_input_info
 from .run import run
 
 try:
@@ -13,3 +11,9 @@ try:
     from .reduced_build_config_parser import parse_config
 except ImportError:
     get_logger('tools_python_utils').info('flatbuffers module is not installed. parse_config will not be available')
+
+# see if we can make the pytorch helpers available.
+import importlib.util  # noqa
+have_torch = importlib.util.find_spec("torch")
+if have_torch:
+    from .pytorch_export_helpers import infer_input_info
diff --git a/tools/python/util/__init__append.py b/tools/python/util/__init__append.py
new file mode 100644
index 0000000000..f1d318335a
--- /dev/null
+++ b/tools/python/util/__init__append.py
@@ -0,0 +1,5 @@
+# appended to the __init__.py in the onnxruntime module's 'tools' folder from /tools/python/util/__init__append.py
+import importlib.util
+have_torch = importlib.util.find_spec("torch")
+if have_torch:
+    from .pytorch_export_helpers import infer_input_info  # noqa
diff --git a/tools/python/util/check_onnx_model_mobile_usability.py b/tools/python/util/check_onnx_model_mobile_usability.py
index 3f62c53b9f..7042d4cd2d 100644
--- a/tools/python/util/check_onnx_model_mobile_usability.py
+++ b/tools/python/util/check_onnx_model_mobile_usability.py
@@ -42,17 +42,18 @@ def check_usability():
     try_eps = usability_checker.analyze_model(args.model_path, skip_optimize=False, logger=logger)
     check_model_can_use_ort_mobile_pkg.run_check(args.model_path, args.config_path, logger)
 
-    logger.info("Run `python -m onnxruntime.tools.convert_onnx_models_to_ort ...` to convert the ONNX model to "
-                "ORT format. By default, the conversion tool will create an ORT format model optimized to "
-                "'basic' level (with a .basic.ort file extension) for use with NNAPI or CoreML, "
-                "and an ORT format model optimized to 'all' level (with a .all.ort file extension) for use with "
-                "the CPU EP.")
+    logger.info("Run `python -m onnxruntime.tools.convert_onnx_models_to_ort ...` to convert the ONNX model to ORT "
+                "format. "
+                "By default, the conversion tool will create an ORT format model with saved optimizations which can "
+                "potentially be applied at runtime (with a .with_runtime_opt.ort file extension) for use with NNAPI "
+                "or CoreML, and a fully optimized ORT format model (with a .ort file extension) for use with the CPU "
+                "EP.")
     if try_eps:
         logger.info("As NNAPI or CoreML may provide benefits with this model it is recommended to compare the "
-                    "performance of the <model>.basic.ort model using the NNAPI EP on Android, and the "
-                    "CoreML EP on iOS, against the performance of the <model>.all.ort model using the CPU EP.")
+                    "performance of the <model>.with_runtime_opt.ort model using the NNAPI EP on Android, and the "
+                    "CoreML EP on iOS, against the performance of the <model>.ort model using the CPU EP.")
     else:
-        logger.info("For optimal performance the <model>.all.ort model should be used with the CPU EP. ")
+        logger.info("For optimal performance the <model>.ort model should be used with the CPU EP. ")
 
 
 if __name__ == '__main__':
diff --git a/tools/python/util/convert_onnx_models_to_ort.py b/tools/python/util/convert_onnx_models_to_ort.py
index 48c7d6552b..0645bac1b4 100644
--- a/tools/python/util/convert_onnx_models_to_ort.py
+++ b/tools/python/util/convert_onnx_models_to_ort.py
@@ -3,44 +3,37 @@
 # Licensed under the MIT License.
 
 import argparse
+import contextlib
+import enum
 import os
 import pathlib
+import tempfile
 import typing
 
 import onnxruntime as ort
-from .ort_format_model import create_config_from_models
+from .file_utils import files_from_file_or_dir, path_match_suffix_ignore_case
 from .onnx_model_utils import get_optimization_level
+from .ort_format_model import create_config_from_models
 
 
-def _path_match_suffix_ignore_case(path: typing.Union[pathlib.Path, str], suffix: str):
-    if not isinstance(path, str):
-        path = str(path)
-    return path.casefold().endswith(suffix.casefold())
+class OptimizationStyle(enum.Enum):
+    Fixed = 0
+    Runtime = 1
 
 
-def _onnx_model_path_to_ort_model_path(onnx_model_path: pathlib.Path, optimization_level_str: str):
-    assert onnx_model_path.is_file() and _path_match_suffix_ignore_case(onnx_model_path, ".onnx")
-    return onnx_model_path.with_suffix(".{}.ort".format(optimization_level_str))
+def _optimization_suffix(optimization_style: OptimizationStyle, suffix: str):
+    return "{}{}".format(".with_runtime_opt" if optimization_style == OptimizationStyle.Runtime else "",
+                         suffix)
 
 
-def _create_config_file_from_ort_models(onnx_model_path_or_dir: pathlib.Path, optimization_level: str,
-                                        enable_type_reduction: bool):
-    if onnx_model_path_or_dir.is_dir():
-        # model directory
-        model_path_or_dir = onnx_model_path_or_dir
-        config_path = None  # default path in model directory
-    else:
-        # single model
-        model_path_or_dir = _onnx_model_path_to_ort_model_path(onnx_model_path_or_dir, optimization_level)
-        suffix = f'.{optimization_level}.config'
-        config_suffix = ".{}{}".format(
-            'required_operators_and_types' if enable_type_reduction else 'required_operators', suffix)
-        config_path = model_path_or_dir.with_suffix(config_suffix)
-
-    create_config_from_models(model_path_or_dir=str(model_path_or_dir),
-                              output_file=str(config_path) if config_path is not None else None,
-                              enable_type_reduction=enable_type_reduction,
-                              optimization_level=optimization_level)
+def _create_config_file_path(model_path_or_dir: pathlib.Path,
+                             optimization_style: OptimizationStyle,
+                             enable_type_reduction: bool):
+    config_name = "{}{}".format('required_operators_and_types' if enable_type_reduction else 'required_operators',
+                                _optimization_suffix(optimization_style, ".config"))
+    if model_path_or_dir.is_dir():
+        return model_path_or_dir / config_name
+    return model_path_or_dir.with_suffix(f".{config_name}")
 
 
 def _create_session_options(optimization_level: ort.GraphOptimizationLevel,
@@ -60,31 +53,33 @@ def _create_session_options(optimization_level: ort.GraphOptimizationLevel,
     return so
 
 
-def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_nnapi: bool, use_coreml: bool,
+def _convert(model_path_or_dir: pathlib.Path, output_dir: typing.Optional[pathlib.Path],
+             optimization_level_str: str, optimization_style: OptimizationStyle,
              custom_op_library: pathlib.Path, create_optimized_onnx_model: bool, allow_conversion_failures: bool,
-             target_platform: str, session_options_config_entries: typing.Dict[str, str]):
+             target_platform: str, session_options_config_entries: typing.Dict[str, str]) \
+        -> typing.List[pathlib.Path]:
+
+    model_dir = model_path_or_dir if model_path_or_dir.is_dir() else model_path_or_dir.parent
+    output_dir = output_dir or model_dir
 
     optimization_level = get_optimization_level(optimization_level_str)
 
-    models = []
-    if model_path_or_dir.is_file() and _path_match_suffix_ignore_case(model_path_or_dir, ".onnx"):
-        models.append(model_path_or_dir)
-    elif model_path_or_dir.is_dir():
-        for root, _, files in os.walk(model_path_or_dir):
-            for file in files:
-                if _path_match_suffix_ignore_case(file, ".onnx"):
-                    models.append(pathlib.Path(root, file))
+    def is_model_file_to_convert(file_path: pathlib.Path):
+        if not path_match_suffix_ignore_case(file_path, ".onnx"):
+            return False
+        # ignore any files with an extension of .optimized.onnx which are presumably from previous executions
+        # of this script
+        if path_match_suffix_ignore_case(file_path, ".optimized.onnx"):
+            print(f"Ignoring '{file_path}'")
+            return False
+        return True
+
+    models = files_from_file_or_dir(model_path_or_dir, is_model_file_to_convert)
 
     if len(models) == 0:
-        raise ValueError("No .onnx files were found in '{}'".format(model_path_or_dir))
+        raise ValueError("No model files were found in '{}'".format(model_path_or_dir))
 
     providers = ['CPUExecutionProvider']
-    if use_nnapi:
-        # providers are priority based, so register NNAPI first
-        providers.insert(0, 'NnapiExecutionProvider')
-    if use_coreml:
-        # providers are priority based, so register CoreML first
-        providers.insert(0, 'CoreMLExecutionProvider')
 
     # if the optimization level is 'all' we manually exclude the NCHWc transformer. It's not applicable to ARM
     # devices, and creates a device specific model which won't run on all hardware.
@@ -94,26 +89,29 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_n
     if optimization_level == ort.GraphOptimizationLevel.ORT_ENABLE_ALL and target_platform != 'amd64':
         optimizer_filter = ['NchwcTransformer']
 
-    num_failures = 0
+    converted_models = []
 
     for model in models:
         try:
 
-            # ignore any files with an extension of .optimized.onnx which are presumably from previous executions
-            # of this script
-            if _path_match_suffix_ignore_case(model, ".optimized.onnx"):
-                print("Ignoring '{}'".format(model))
-                continue
+            relative_model_path = model.relative_to(model_dir)
 
-            # create .ort file in same dir as original onnx model
-            ort_target_path = _onnx_model_path_to_ort_model_path(model, optimization_level_str)
+            (output_dir / relative_model_path).parent.mkdir(parents=True, exist_ok=True)
+
+            ort_target_path = (output_dir / relative_model_path).with_suffix(
+                _optimization_suffix(optimization_style, ".ort"))
 
             if create_optimized_onnx_model:
-                # Create an ONNX file with the same optimizations that will be used for the ORT format file.
+                # Create an ONNX file with the same optimization level that will be used for the ORT format file.
                 # This allows the ONNX equivalent of the ORT format model to be easily viewed in Netron.
-                optimized_target_path = model.with_suffix(".{}.optimized.onnx".format(optimization_level_str))
+                # If runtime optimizations are saved in the ORT format model, there may be some difference in the
+                # graphs at runtime between the ORT format model and this saved ONNX model.
+                optimized_target_path = (output_dir / relative_model_path).with_suffix(".optimized.onnx")
                 so = _create_session_options(optimization_level, optimized_target_path, custom_op_library,
                                              session_options_config_entries)
+                if optimization_style == OptimizationStyle.Runtime:
+                    # Limit the optimizations to those that can run in a model with runtime optimizations.
+                    so.add_session_config_entry('optimization.minimal_build_optimizations', 'apply')
 
                 print("Saving optimized ONNX model {} to {}".format(model, optimized_target_path))
                 _ = ort.InferenceSession(str(model), sess_options=so, providers=providers,
@@ -123,11 +121,15 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_n
             so = _create_session_options(optimization_level, ort_target_path, custom_op_library,
                                          session_options_config_entries)
             so.add_session_config_entry('session.save_model_format', 'ORT')
+            if optimization_style == OptimizationStyle.Runtime:
+                so.add_session_config_entry('optimization.minimal_build_optimizations', 'save')
 
             print("Converting optimized ONNX model {} to ORT format model {}".format(model, ort_target_path))
             _ = ort.InferenceSession(str(model), sess_options=so, providers=providers,
                                      disabled_optimizers=optimizer_filter)
 
+            converted_models.append(ort_target_path)
+
             # orig_size = os.path.getsize(onnx_target_path)
             # new_size = os.path.getsize(ort_target_path)
             # print("Serialized {} to {}. Sizes: orig={} new={} diff={} new:old={:.4f}:1.0".format(
@@ -136,9 +138,10 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_n
             print("Error converting {}: {}".format(model, e))
             if not allow_conversion_failures:
                 raise
-            num_failures += 1
 
-    print("Converted {} models. {} failures.".format(len(models), num_failures))
+    print("Converted {}/{} models successfully.".format(len(converted_models), len(models)))
+
+    return converted_models
 
 
 def parse_args():
@@ -146,38 +149,28 @@ def parse_args():
         os.path.basename(__file__),
         description='''Convert the ONNX format model/s in the provided directory to ORT format models.
         All files with a `.onnx` extension will be processed. For each one, an ORT format model will be created in the
-        same directory. A configuration file will also be created called `required_operators.config`, and will contain
-        the list of required operators for all converted models.
-        This configuration file should be used as input to the minimal build via the `--include_ops_by_config`
-        parameter.
+        same directory. A configuration file will also be created containing the list of required operators for all
+        converted models. This configuration file should be used as input to the minimal build via the
+        `--include_ops_by_config` parameter.
         '''
     )
 
-    parser.add_argument('--use_nnapi', action='store_true',
-                        help='Enable the NNAPI Execution Provider when creating models and determining required '
-                             'operators. Note that this will limit the optimizations possible on nodes that the '
-                             'NNAPI execution provider takes, in order to preserve those nodes in the ORT format '
-                             'model.')
-
-    parser.add_argument('--use_coreml', action='store_true',
-                        help='Enable the CoreML Execution Provider when creating models and determining required '
-                             'operators. Note that this will limit the optimizations possible on nodes that the '
-                             'CoreML execution provider takes, in order to preserve those nodes in the ORT format '
-                             'model.')
-
-    parser.add_argument('--optimization_level', default=['basic', 'all'], nargs='+',
-                        choices=['disable', 'basic', 'extended', 'all'],
-                        help="Level to optimize ONNX model with, prior to converting to ORT format model. "
-                             "These map to the onnxruntime.GraphOptimizationLevel values. "
-                             "If the level is 'all' the NCHWc transformer is manually disabled as it contains device "
-                             "specific logic, so the ORT format model must be generated on the device it will run on. "
-                             "Additionally, the NCHWc optimizations are not applicable to ARM devices. "
-                             "Multiple values can be provided. A model produced with 'all' is optimal for usage with "
-                             "just the CPU Execution Provider. A model produced with 'basic' is required for usage "
-                             "with the NNAPI or CoreML Execution Providers. "
-                             "The filename for the ORT format model will contain the optimization level that was used "
-                             "to create it."
-                        )
+    parser.add_argument('--optimization_style',
+                        nargs='+',
+                        default=[OptimizationStyle.Fixed.name, OptimizationStyle.Runtime.name],
+                        choices=[e.name for e in OptimizationStyle],
+                        help="Style of optimization to perform on the ORT format model. "
+                             "Multiple values may be provided. The conversion will run once for each value. "
+                             "The general guidance is to use models optimized with "
+                             f"'{OptimizationStyle.Runtime.name}' style when using NNAPI or CoreML and "
+                             f"'{OptimizationStyle.Fixed.name}' style otherwise. "
+                             f"'{OptimizationStyle.Fixed.name}': Run optimizations directly before saving the ORT "
+                             "format model. This bakes in any platform-specific optimizations. "
+                             f"'{OptimizationStyle.Runtime.name}': Run basic optimizations directly and save certain "
+                             "other optimizations to be applied at runtime if possible. This is useful when using a "
+                             "compiling EP like NNAPI or CoreML that may run an unknown (at model conversion time) "
+                             "number of nodes. The saved optimizations can further optimize nodes not assigned to the "
+                             "compiling EP at runtime.")
 
     parser.add_argument('--enable_type_reduction', action='store_true',
                         help='Add operator specific type information to the configuration file to potentially reduce '
@@ -188,7 +181,7 @@ def parse_args():
 
     parser.add_argument('--save_optimized_onnx_model', action='store_true',
                         help='Save the optimized version of each ONNX model. '
-                             'This will have the same optimizations applied as the ORT format model.')
+                             'This will have the same level of optimizations applied as the ORT format model.')
 
     parser.add_argument('--allow_conversion_failures', action='store_true',
                         help='Whether to proceed after encountering model conversion failures.')
@@ -200,13 +193,14 @@ def parse_args():
 
     parser.add_argument('--target_platform', type=str, default=None, choices=['arm', 'amd64'],
                         help='Specify the target platform where the exported model will be used. '
-                             'This parameter can be used to choose between platform specific options, '
-                             'such as QDQIsInt8Allowed(arm), NCHWc (amd64) and NHWC (arm/amd64) format different '
-                             'optimizer level options,etc.')
+                             'This parameter can be used to choose between platform-specific options, '
+                             'such as QDQIsInt8Allowed(arm), NCHWc (amd64) and NHWC (arm/amd64) format, different '
+                             'optimizer level options, etc.')
 
     parser.add_argument('model_path_or_dir', type=pathlib.Path,
                         help='Provide path to ONNX model or directory containing ONNX model/s to convert. '
-                             'All files with a .onnx extension, including in subdirectories, will be processed.')
+                             'All files with a .onnx extension, including those in subdirectories, will be '
+                             'processed.')
 
     return parser.parse_args()
 
@@ -214,6 +208,8 @@ def parse_args():
 def convert_onnx_models_to_ort():
     args = parse_args()
 
+    optimization_styles = [OptimizationStyle[style_str] for style_str in args.optimization_style]
+    optimization_level_str = 'all'
     model_path_or_dir = args.model_path_or_dir.resolve()
     custom_op_library = args.custom_op_library.resolve() if args.custom_op_library else None
 
@@ -223,12 +219,6 @@ def convert_onnx_models_to_ort():
     if custom_op_library and not custom_op_library.is_file():
         raise FileNotFoundError("Unable to find custom operator library '{}'".format(custom_op_library))
 
-    if args.use_nnapi and 'NnapiExecutionProvider' not in ort.get_available_providers():
-        raise ValueError('The NNAPI Execution Provider was not included in this build of ONNX Runtime.')
-
-    if args.use_coreml and 'CoreMLExecutionProvider' not in ort.get_available_providers():
-        raise ValueError('The CoreML Execution Provider was not included in this build of ONNX Runtime.')
-
     session_options_config_entries = {}
 
     if args.nnapi_partitioning_stop_ops is not None:
@@ -239,13 +229,49 @@ def convert_onnx_models_to_ort():
     else:
         session_options_config_entries["session.qdqisint8allowed"] = "0"
 
-    for optimization_level in args.optimization_level:
-        print(f"Converting models and creating configuration file for optimization level '{optimization_level}'")
-        _convert(model_path_or_dir, optimization_level, args.use_nnapi, args.use_coreml, custom_op_library,
-                 args.save_optimized_onnx_model, args.allow_conversion_failures, args.target_platform,
-                 session_options_config_entries)
+    for optimization_style in optimization_styles:
+        print("Converting models with optimization style '{}' and level '{}'".format(
+            optimization_style.name, optimization_level_str))
 
-        _create_config_file_from_ort_models(model_path_or_dir, optimization_level, args.enable_type_reduction)
+        converted_models = _convert(
+            model_path_or_dir=model_path_or_dir, output_dir=None,
+            optimization_level_str=optimization_level_str, optimization_style=optimization_style,
+            custom_op_library=custom_op_library,
+            create_optimized_onnx_model=args.save_optimized_onnx_model,
+            allow_conversion_failures=args.allow_conversion_failures,
+            target_platform=args.target_platform,
+            session_options_config_entries=session_options_config_entries)
+
+        with contextlib.ExitStack() as context_stack:
+            if optimization_style == OptimizationStyle.Runtime:
+                # Convert models again without runtime optimizations.
+                # Runtime optimizations may not end up being applied, so we need to use both converted models with and
+                # without runtime optimizations to get a complete set of ops that may be needed for the config file.
+                model_dir = model_path_or_dir if model_path_or_dir.is_dir() else model_path_or_dir.parent
+                temp_output_dir = context_stack.enter_context(
+                    tempfile.TemporaryDirectory(dir=model_dir, suffix=".without_runtime_opt"))
+                session_options_config_entries_for_second_conversion = session_options_config_entries.copy()
+                # Limit the optimizations to those that can run in a model with runtime optimizations.
+                session_options_config_entries_for_second_conversion[
+                    "optimization.minimal_build_optimizations"] = "apply"
+
+                print("Converting models again without runtime optimizations to generate a complete config file. "
+                      "These converted models are temporary and will be deleted.")
+                converted_models += _convert(
+                    model_path_or_dir=model_path_or_dir, output_dir=temp_output_dir,
+                    optimization_level_str=optimization_level_str, optimization_style=OptimizationStyle.Fixed,
+                    custom_op_library=custom_op_library,
+                    create_optimized_onnx_model=False,  # not useful as they would be created in a temp directory
+                    allow_conversion_failures=args.allow_conversion_failures,
+                    target_platform=args.target_platform,
+                    session_options_config_entries=session_options_config_entries_for_second_conversion)
+
+            print("Generating config file from ORT format models with optimization style '{}' and level '{}'".format(
+                optimization_style.name, optimization_level_str))
+
+            config_file = _create_config_file_path(model_path_or_dir, optimization_style, args.enable_type_reduction)
+
+            create_config_from_models(converted_models, config_file, args.enable_type_reduction)
 
 
 if __name__ == '__main__':
diff --git a/tools/python/util/file_utils.py b/tools/python/util/file_utils.py
new file mode 100644
index 0000000000..73505b7336
--- /dev/null
+++ b/tools/python/util/file_utils.py
@@ -0,0 +1,46 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import pathlib
+import typing
+import os
+
+
+def path_match_suffix_ignore_case(path: typing.Union[pathlib.Path, str], suffix: str) -> bool:
+    '''
+    Returns whether `path` ends in `suffix`, ignoring case.
+    '''
+    if not isinstance(path, str):
+        path = str(path)
+    return path.casefold().endswith(suffix.casefold())
+
+
+def files_from_file_or_dir(file_or_dir_path: typing.Union[pathlib.Path, str],
+                           predicate: typing.Callable[[pathlib.Path], bool] = lambda _: True) \
+        -> typing.List[pathlib.Path]:
+    '''
+    Gets the files in `file_or_dir_path` satisfying `predicate`.
+    If `file_or_dir_path` is a file, the single file is considered. Otherwise, all files in the directory are
+    considered.
+    :param file_or_dir_path: Path to a file or directory.
+    :param predicate: Predicate to determine if a file is included.
+    :return: A list of files.
+    '''
+    if not isinstance(file_or_dir_path, pathlib.Path):
+        file_or_dir_path = pathlib.Path(file_or_dir_path)
+
+    selected_files = []
+
+    def process_file(file_path: pathlib.Path):
+        if predicate(file_path):
+            selected_files.append(file_path)
+
+    if file_or_dir_path.is_dir():
+        for root, _, files in os.walk(file_or_dir_path):
+            for file in files:
+                file_path = pathlib.Path(root, file)
+                process_file(file_path)
+    else:
+        process_file(file_or_dir_path)
+
+    return selected_files
diff --git a/tools/python/util/onnx_model_utils.py b/tools/python/util/onnx_model_utils.py
index d232324d04..6d51d0b5da 100644
--- a/tools/python/util/onnx_model_utils.py
+++ b/tools/python/util/onnx_model_utils.py
@@ -124,6 +124,34 @@ def _replace_symbolic_dim_value(graph: onnx.GraphProto, **kwargs):
     update_dim_values(graph.value_info)
 
 
+def _remove_invalid_dim_values_impl(graph: onnx.GraphProto):
+    def clear_invalid_values(value):
+        if value.type.HasField("tensor_type"):
+            shape = value.type.tensor_type.shape
+            if shape:
+                for dim in shape.dim:
+                    if dim.HasField('dim_value') and dim.dim_value < 1:
+                        dim.Clear()
+
+    for i in graph.input:
+        clear_invalid_values(i)
+
+    for o in graph.output:
+        clear_invalid_values(o)
+
+    for vi in graph.value_info:
+        clear_invalid_values(vi)
+
+
+def remove_invalid_dim_values(graph: onnx.GraphProto):
+    '''
+    Iterate the graph and subgraphs, unsetting any dim_value entries that have a value of less than 1.
+    These are typically erroneously inserted by a converter to represent a dynamic dimension.
+    :param graph: GraphProto to update
+    '''
+    iterate_graph_per_graph_func(graph, _remove_invalid_dim_values_impl)
+
+
 def make_dim_param_fixed(graph: onnx.GraphProto, param_name: str, value: int):
     '''
     Iterate all values in the graph, replacing dim_param in a tensor shape with the provided value.
@@ -144,6 +172,9 @@ def make_input_shape_fixed(graph: onnx.GraphProto, input_name: str, fixed_shape:
     :param fixed_shape: Shape to use.
     '''
 
+    # remove any invalid dim values first. typically this is a dim_value of -1.
+    remove_invalid_dim_values(graph)
+
     for i in graph.input:
         if i.name == input_name:
             if not i.type.HasField("tensor_type"):
diff --git a/tools/python/util/ort_format_model/utils.py b/tools/python/util/ort_format_model/utils.py
index a6d3c2c868..2be004dc9c 100644
--- a/tools/python/util/ort_format_model/utils.py
+++ b/tools/python/util/ort_format_model/utils.py
@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-import os
+import pathlib
 import typing
 
 from .operator_type_usage_processors import OperatorTypeUsageManager
@@ -11,72 +11,36 @@ from ..logger import get_logger
 log = get_logger("ort_format_model.utils")
 
 
-def _extract_ops_and_types_from_ort_models(model_path_or_dir: str, enable_type_reduction: bool,
-                                           optimization_level: str = None):
-    if not os.path.exists(model_path_or_dir):
-        raise ValueError('Path to model/s does not exist: {}'.format(model_path_or_dir))
-
+def _extract_ops_and_types_from_ort_models(model_files: typing.Iterable[pathlib.Path], enable_type_reduction: bool):
     required_ops = {}
     op_type_usage_manager = OperatorTypeUsageManager() if enable_type_reduction else None
 
-    suffix = f'.{optimization_level}.ort' if optimization_level else '.ort'
-    if os.path.isfile(model_path_or_dir):
-        if model_path_or_dir.lower().endswith(suffix):
-            model_processor = OrtFormatModelProcessor(model_path_or_dir, required_ops, op_type_usage_manager)
-            model_processor.process()  # this updates required_ops and op_type_processors
-            log.info('Processed {}'.format(model_path_or_dir))
-        else:
-            log.debug('Skipped {}'.format(model_path_or_dir))
-    else:
-        for root, _, files in os.walk(model_path_or_dir):
-            for file in files:
-                model_path = os.path.join(root, file)
-                if file.lower().endswith(suffix):
-                    model_processor = OrtFormatModelProcessor(model_path, required_ops, op_type_usage_manager)
-                    model_processor.process()  # this updates required_ops and op_type_processors
-                    log.info('Processed {}'.format(model_path))
-                else:
-                    log.debug('Skipped {}'.format(model_path))
+    for model_file in model_files:
+        if not model_file.is_file():
+            raise ValueError(f"Path is not a file: '{model_file}'")
+        model_processor = OrtFormatModelProcessor(str(model_file), required_ops, op_type_usage_manager)
+        model_processor.process()  # this updates required_ops and op_type_processors
 
     return required_ops, op_type_usage_manager
 
 
-def create_config_from_models(model_path_or_dir: str, output_file: str = None, enable_type_reduction: bool = True,
-                              optimization_level: typing.Optional[str] = None):
+def create_config_from_models(model_files: typing.Iterable[pathlib.Path], output_file: pathlib.Path,
+                              enable_type_reduction: bool):
     '''
     Create a configuration file with required operators and optionally required types.
-    :param model_path_or_dir: Path to recursively search for ORT format models, or to a single ORT format model.
+    :param model_files: Model files to use to generate the configuration file.
     :param output_file: File to write configuration to.
-                        Defaults to creating required_operators[_and_types].config in the model_path_or_dir directory.
     :param enable_type_reduction: Include required type information for individual operators in the configuration.
-    :param optimization_level: Filter files and adjust default output_file based on the optimization level. If set,
-           looks for '.<optimization_level>.ort' as the file suffix. Uses '.<optimization_level>.config' as the config
-           file suffix.
-           When we convert models we include the optimization level in the filename. When creating the configuration
-           we only want to create it for the specific optimization level so that we don't include irrelevant operators.
     '''
 
-    required_ops, op_type_processors = _extract_ops_and_types_from_ort_models(model_path_or_dir, enable_type_reduction,
-                                                                              optimization_level)
+    required_ops, op_type_processors = _extract_ops_and_types_from_ort_models(model_files, enable_type_reduction)
 
-    if output_file:
-        directory, filename = os.path.split(output_file)
-        if not filename:
-            raise RuntimeError("Invalid output path for configuration: {}".format(output_file))
-
-        if directory and not os.path.exists(directory):
-            os.makedirs(directory)
-    else:
-        dir = model_path_or_dir
-        if os.path.isfile(model_path_or_dir):
-            dir = os.path.dirname(model_path_or_dir)
-
-        suffix = f'.{optimization_level}.config' if optimization_level else '.config'
-        output_file = os.path.join(
-            dir, ('required_operators_and_types' if enable_type_reduction else 'required_operators') + suffix)
+    output_file.parent.mkdir(parents=True, exist_ok=True)
 
     with open(output_file, 'w') as out:
-        out.write("# Generated from model/s in {}\n".format(model_path_or_dir))
+        out.write("# Generated from model/s:\n")
+        for model_file in sorted(model_files):
+            out.write(f"# - {model_file}\n")
 
         for domain in sorted(required_ops.keys()):
             for opset in sorted(required_ops[domain].keys()):