diff --git a/cmake/onnxruntime_csharp.cmake b/cmake/onnxruntime_csharp.cmake
index 58a9f894ec..c65acd849f 100644
--- a/cmake/onnxruntime_csharp.cmake
+++ b/cmake/onnxruntime_csharp.cmake
@@ -50,6 +50,10 @@ if (onnxruntime_USE_XNNPACK)
   STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_XNNPACK;")
 endif()
 
+if (onnxruntime_ENABLE_TRAINING_ON_DEVICE)
+  STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "__TRAINING_ENABLED_NATIVE_BUILD__;")
+endif()
+
 include(CSharpUtilities)
 
 # generate Directory.Build.props
diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj
index 94635f2dc3..5473246e8b 100644
--- a/csharp/OnnxRuntime.CSharp.proj
+++ b/csharp/OnnxRuntime.CSharp.proj
@@ -1,12 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!--
-This is the master msbuild project file for all csharp components. 
+This is the master msbuild project file for all csharp components.
 This is created so that the NuGet dependencies are restored before the projects are built during a CI build.
 CMake creates a target to this project
 -->
 
 <Project DefaultTargets="Build">
-  
+
   <PropertyGroup>
     <OrtPackageId Condition=" '$(OrtPackageId)' == '' ">Microsoft.ML.OnnxRuntime</OrtPackageId>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -54,7 +54,7 @@ CMake creates a target to this project
    <Target Name="ObtainPackageVersion" BeforeTargets="Build;CreatePackage;CreateWindowsAIPackage">
     <ReadLinesFromFile File="..\VERSION_NUMBER">
       <Output TaskParameter="Lines" ItemName="MajorVersionNumber"/>
-    </ReadLinesFromFile>   
+    </ReadLinesFromFile>
     <Exec Command="git rev-parse --short HEAD" ConsoleToMSBuild="true">
         <Output TaskParameter="ConsoleOutput" PropertyName="GitCommitHashShort" />
     </Exec>
@@ -72,7 +72,7 @@ CMake creates a target to this project
     </ConvertToAbsolutePath>
     <ConvertToAbsolutePath Paths="$(OnnxRuntimeSourceDirectory)">
       <Output TaskParameter="AbsolutePaths" PropertyName="OnnxRuntimeSourceDirectoryAbs"/>
-    </ConvertToAbsolutePath>    
+    </ConvertToAbsolutePath>
     <PropertyGroup>
       <CurrentDate>$([System.DateTime]::UtcNow.ToString(yyyyMMdd))</CurrentDate>
       <CurrentTime>$([System.DateTime]::UtcNow.ToString(hhmm))</CurrentTime>
@@ -82,12 +82,12 @@ CMake creates a target to this project
     </PropertyGroup>
     <Message Importance="High" Text="PackageVersion=$(PackageVersion)" />
   </Target>
-  
+
   <Target Name="CreatePackage">
     <Message Importance="High" Text="Bundling managed assemblies into a NuGet package ..." />
     <MSBuild Projects="src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
              Targets="CopyMiscFiles;Pack"
-            Properties="NoBuild=true;Platform=AnyCPU;PackageVersion=$(PackageVersion);OrtPackageId=$(OrtPackageId);SelectedTargets=All"/> 
+            Properties="NoBuild=true;Platform=AnyCPU;PackageVersion=$(PackageVersion);OrtPackageId=$(OrtPackageId);SelectedTargets=All"/>
 
     <Message Importance="High" Text="Generating nuspec for the native Nuget package ..." />
     <Exec ContinueOnError="False" Command="python $(GenerateNuspecScript) --package_version $(PackageVersion) --package_name $(OrtPackageId) --target_architecture $(TargetArchitecture) --build_config $(Configuration) --native_build_path $(NativeBuildOutputDirAbs) --packages_path $(OnnxRuntimePackagesDirectoryAbs) --ort_build_path $(OnnxRuntimeBuildDirectoryAbs) --sources_path $(OnnxRuntimeSourceDirectoryAbs) --commit_id $(GitCommitHash) --is_release_build $(IsReleaseBuild) --execution_provider $(ExecutionProvider)" ConsoleToMSBuild="true">
@@ -105,7 +105,7 @@ CMake creates a target to this project
 
     <Copy
     SourceFiles="$(NativeBuildOutputDirAbs)\$(OrtPackageId).$(PackageVersion).nupkg"
-    DestinationFolder="$(NativeBuildOutputDirAbs)\nuget-artifacts"
+    DestinationFolder="$(NativeBuildOutputDirAbs)\nuget-local-artifacts"
     />
   </Target>
 
@@ -113,7 +113,7 @@ CMake creates a target to this project
     <LicenseFile Include="$(OnnxRuntimeSourceDirectory)\LICENSE"/>
   </ItemGroup>
 
-  <Target Name="CreateWindowsAIPackage">  
+  <Target Name="CreateWindowsAIPackage">
     <Copy SourceFiles="@(LicenseFile)" DestinationFiles="@(LicenseFile->'$(OnnxRuntimeSourceDirectory)\%(Filename).txt')"/>
 
     <Message Importance="High" Text="Generating nuspec for the Microsoft.AI.MachineLearning Nuget package ..." />
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
index 679361c336..e6a8272fd8 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
@@ -4,19 +4,19 @@
     <OrtPackageId Condition="'$(OrtPackageId)' == ''">Microsoft.ML.OnnxRuntime</OrtPackageId>
   </PropertyGroup>
 
-  <!-- 
+  <!--
   Temporary setup until official Visual Studio 2022 release supports .net6, as the CIs require the official release.
   We will be able to build all targets with VS once that happens.
- 
+
   Until then, we need to build the pre-.net6 targets with VS and the .net6 targets with dotnet.
-  
+
   The pre-.net6 Xamarin targets are optional and only included if the machine has the required workloads.
 
   We have 3 scenarios
     1) Build pre-net6 targets with VS - SelectedTargets=PreNet6
     2) Build net6 targets - SelectedTargets=Net6
     3) Run 'Pack' target to create nuget package from combination of 1 and 2 - SelectedTargets=All
-  
+
   Default is PreNet6 so that existing projects and CI builds will do the same thing unless explicitly updated.
   -->
   <PropertyGroup>
@@ -24,18 +24,18 @@
     <BaseTargets>netstandard1.1;netstandard2.0;net5.0;netcoreapp3.1</BaseTargets>
   </PropertyGroup>
 
-  <!-- only set the Xamarin mobile targets if we're building an ORT package, 
+  <!-- only set the Xamarin mobile targets if we're building an ORT package,
        and only if the mobile workloads are installed -->
-  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR 
-                             '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu') AND 
-                             Exists('$(MSBuildExtensionsPath)\Xamarin\Android') AND 
+  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR
+                             '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu') AND
+                             Exists('$(MSBuildExtensionsPath)\Xamarin\Android') AND
                              Exists('$(MSBuildExtensionsPath)\Xamarin\iOS')">
     <XamarinTargets>xamarinios10;monoandroid11.0</XamarinTargets>
   </PropertyGroup>
 
   <!-- only set the .net6 targets if we're building an ORT package.
        we can add .net6 support to other packages later as needed -->
-  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR 
+  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR
                              '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu')">
     <Net6Targets>net6.0;net6.0-android;net6.0-ios;net6.0-macos</Net6Targets>
   </PropertyGroup>
@@ -64,7 +64,7 @@
     <!--internal build related properties-->
     <OnnxRuntimeRoot>$(ProjectDir)..\..\..</OnnxRuntimeRoot>
     <OnnxRuntimeCsharpRoot>$(OnnxRuntimeRoot)\csharp</OnnxRuntimeCsharpRoot>
-    <TargetArchitecture Condition=" '$(TargetArchitecture)' == '' ">x64</TargetArchitecture>  
+    <TargetArchitecture Condition=" '$(TargetArchitecture)' == '' ">x64</TargetArchitecture>
 
     <RootNamespace>Microsoft.ML.OnnxRuntime</RootNamespace>
     <AssemblyName>Microsoft.ML.OnnxRuntime</AssemblyName>
@@ -72,11 +72,15 @@
     <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
     <DebugType>portable</DebugType>
 
-    <!-- Training build property. With Visual Studio, change to true to enable. -->
-    <OnDeviceTraining Condition="'$(OnDeviceTraining)' == ''">false</OnDeviceTraining>
+    <!-- Controls whether C# Bindings for Training are included in the Managed Nuget Package.
+    Bindings for training are included by default. If user tries to call training apis when the native package installed
+    on their device is not built for training, an exception will be thrown with the following message -
+    "Training is disabled in the current build. Please build onnxruntime from source with the build flags
+    enable_training and enable_training_on_device. "-->
+   <OnDeviceTraining Condition="'$(OnDeviceTraining)' == ''">true</OnDeviceTraining>
 
     <!--- The package name is always hardcoded as the package created by this project only contains managed assemblies -->
-    <!--- The parameter OrtPackageId is only used for some conditional logic below -->    
+    <!--- The parameter OrtPackageId is only used for some conditional logic below -->
     <PackageId>Microsoft.ML.OnnxRuntime.Managed</PackageId>
     <Authors>Microsoft</Authors>
     <PackageVersion Condition=" '$(PackageVersion)' == '' And '$(Configuration)' == 'Debug' ">1.0.0</PackageVersion>
@@ -105,27 +109,35 @@
     <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
     <Configurations>Debug;Release;RelWithDebInfo</Configurations>
 
-    <IsLinuxBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinuxBuild> 
-    <IsWindowsBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindowsBuild> 
+    <IsLinuxBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinuxBuild>
+    <IsWindowsBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindowsBuild>
     <IsMacOSBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsMacOSBuild>
 
-    <!-- $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) gives better results than 
+    <!-- $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) gives better results than
          $(TargetPlatformIdentifier). See https://github.com/dotnet/msbuild/issues/7359
          Note there are slight differences in casing (e.g. macos vs macOS), so if we ever
          change to use $(TargetPlatformIdentifier) we need to adjust for that.
-         
+
     -->
-    <IsXamarinTarget Condition="$(TargetFramework.StartsWith('xamarinios')) OR 
+    <IsXamarinTarget Condition="$(TargetFramework.StartsWith('xamarinios')) OR
                                 $(TargetFramework.StartsWith('monoandroid'))">true</IsXamarinTarget>
-    <IsAndroidTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android' OR 
+    <IsAndroidTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android' OR
                                 $(TargetFramework.StartsWith('monoandroid'))">true</IsAndroidTarget>
-    <IsIOSTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'ios' OR 
+    <IsIOSTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'ios' OR
                             $(TargetFramework.StartsWith('xamarinios'))">true</IsIOSTarget>
     <IsMacTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'macos' OR
                             $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'maccatalyst'">true</IsMacTarget>
   </PropertyGroup>
 
-  <!-- 
+  <!-- Enable training APIs for the build. The native package must be
+       a training build with build flags 'enable_training' and
+       'enable_training_on_device' set.
+  -->
+  <PropertyGroup Condition="'$(OnDeviceTraining)'=='true'">
+    <OrtConstants>$(OrtConstants);__ENABLE_TRAINING_ON_DEVICE__</OrtConstants>
+  </PropertyGroup>
+
+  <!--
     Properties that depend on the system we're building on.
   -->
   <PropertyGroup Condition="'$(IsLinuxBuild)'=='true'">
@@ -146,7 +158,7 @@
       <NativeBuildOutputDir>$(OnnxRuntimeBuildDirectory)\$(Configuration)</NativeBuildOutputDir>
   </PropertyGroup>
 
-  <!-- 
+  <!--
     Properties that depend on the target framework.
   -->
   <PropertyGroup Condition="'$(IsIOSTarget)'=='true' OR '$(IsAndroidTarget)'=='true'">
@@ -179,14 +191,6 @@
     <OrtConstants>$(OrtConstants);__XAMARIN__</OrtConstants>
   </PropertyGroup>
 
-  <!-- Enable training APIs for the build. The native package must be
-       a training build with build flags 'enable_training' and
-       'enable_training_on_device' set.
-  -->
-  <PropertyGroup Condition="'$(OnDeviceTraining)' == 'true'">
-    <OrtConstants>$(OrtConstants);__ENABLE_TRAINING_ON_DEVICE__</OrtConstants>
-  </PropertyGroup>
-
   <PropertyGroup>
     <DefineConstants>$(DefineConstants);$(OrtConstants)</DefineConstants>
   </PropertyGroup>
@@ -287,7 +291,7 @@
     <None Include="targets\netstandard\$(PackageId).targets"
           PackagePath="build\netstandard1.1\$(PackageId).targets;build\netstandard2.0\$(PackageId).targets"
           Pack="true"
-          Visible="false" 
+          Visible="false"
     />
 
     <!-- Some tools to be packaged in nightly build only, should not be released -->
@@ -346,5 +350,5 @@
       DestinationFolder="$(NativeBuildOutputDir)"
     />
   </Target>
-  
+
 </Project>
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs
index 4cff5bccb1..fec2c18572 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs
@@ -33,7 +33,7 @@ namespace Microsoft.ML.OnnxRuntime
             }
             else
             {
-                throw new InvalidOperationException("Training is disabled in the current build");
+                throw new InvalidOperationException("Training is disabled in the current build. Please build ONNXRuntime from source with the build flags enable_training and enable_training_on_device. \n");
             }
         }
 
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs
index b5b5a7c598..8d804628af 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs
@@ -336,7 +336,7 @@ namespace Microsoft.ML.OnnxRuntime
         {
             if (!NativeTrainingMethods.TrainingEnabled())
             {
-                throw new InvalidOperationException("Training is disabled in the current build.");
+                throw new InvalidOperationException("Training is disabled in the current build. Please build ONNXRuntime from source with the build flags enable_training and enable_training_on_device. \n");
             }
             var options = sessOptions;
             if (sessOptions == null)
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs
index f5bf62b4b8..f643efa720 100644
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs
@@ -23,6 +23,17 @@ namespace Microsoft.ML.OnnxRuntime.Tests
             this.output = o;
         }
 
+#if !__TRAINING_ENABLED_NATIVE_BUILD__
+        [Fact(DisplayName = "TestLoadCheckpointThrows")]
+        public void TestLoadCheckpointThrows()
+        {
+            string path = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt");
+            var ex = Assert.Throws<InvalidOperationException>(() => { var opt = new CheckpointState(path); });
+            Assert.Contains("Training is disabled in the current build.", ex.Message);
+        }
+#endif
+
+#if __TRAINING_ENABLED_NATIVE_BUILD__
         [Fact(DisplayName = "TestLoadCheckpoint")]
         public void TestLoadCheckpoint()
         {
@@ -122,13 +133,13 @@ namespace Microsoft.ML.OnnxRuntime.Tests
                 outputs = trainingSession.TrainStep(pinnedInputs);
                 var outputBuffer = outputs.ElementAtOrDefault(0);
 
-                Assert.Equal("542.loss", outputBuffer.Name);
+                Assert.Equal("onnx::loss::21273", outputBuffer.Name);
                 Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, outputBuffer.ValueType);
                 Assert.Equal(TensorElementType.Float, outputBuffer.ElementType);
 
                 var outLabelTensor = outputBuffer.AsTensor<float>();
                 Assert.NotNull(outLabelTensor);
-                Assert.Equal(expectedOutput, outLabelTensor, new FloatComparer());
+                Assert.Equal(expectedOutput, outLabelTensor.ToArray(), new FloatComparer());
             }
         }
 
@@ -304,5 +315,6 @@ namespace Microsoft.ML.OnnxRuntime.Tests
                 return x.GetHashCode();
             }
         }
-    }
+#endif
+        }
 }
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj
index fbb0417215..dafebd9ccb 100644
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj
@@ -17,10 +17,18 @@
     <AssemblyOriginatorKeyFile>..\..\OnnxRuntime.snk</AssemblyOriginatorKeyFile>
     <Configurations>Debug;Release;RelWithDebInfo</Configurations>
     <!-- end -->
-    <DefineConstants>__NET_CORE_APP__;$(DefineConstants)</DefineConstants>
 
-    <!-- Training build property. With Visual Studio, change to true to enable. -->
-    <OnDeviceTraining Condition="'$(OnDeviceTraining)' == ''">false</OnDeviceTraining>
+    <!-- Training build property.
+    Should be set to true when training is enabled in onnxruntime native binary -->
+    <TrainingEnabledNativeBuild Condition="'$(TrainingEnabledNativeBuild)' == ''">false</TrainingEnabledNativeBuild>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TrainingEnabledNativeBuild)'=='true'">
+    <ExtraDefineConstants>$(ExtraDefineConstants);__TRAINING_ENABLED_NATIVE_BUILD__</ExtraDefineConstants>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <DefineConstants>__NET_CORE_APP__;$(DefineConstants);$(ExtraDefineConstants)</DefineConstants>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(IsLinuxBuild)'=='true'">
@@ -95,12 +103,12 @@
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\Tensors\ArrayTensorExtensionsTests.cs">
       <Link>ArrayTensorExtensionsTests.cs</Link>
     </Compile>
-  </ItemGroup>
-
-  <ItemGroup Condition="'$(OnDeviceTraining)' == 'true'">
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\TrainingTest.cs">
       <Link>TrainingTest.cs</Link>
     </Compile>
+  </ItemGroup>
+
+  <ItemGroup>
     <None Include="$(OnnxRuntimeCSharpRoot)\..\onnxruntime\test\testdata\training_api\**\*.*">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <Visible>false</Visible>
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index 6d094c8c6a..a6e6b4cdc7 100644
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -2131,7 +2131,17 @@ def derive_linux_build_property():
 
 
 def build_nuget_package(
-    source_dir, build_dir, configs, use_cuda, use_openvino, use_tensorrt, use_dnnl, use_tvm, use_winml, use_snpe
+    source_dir,
+    build_dir,
+    configs,
+    use_cuda,
+    use_openvino,
+    use_tensorrt,
+    use_dnnl,
+    use_tvm,
+    use_winml,
+    use_snpe,
+    enable_training_on_device,
 ):
     if not (is_windows() or is_linux()):
         raise BuildError(
@@ -2150,7 +2160,12 @@ def build_nuget_package(
     target_name = "/t:CreatePackage"
     execution_provider = '/p:ExecutionProvider="None"'
     package_name = '/p:OrtPackageId="Microsoft.ML.OnnxRuntime"'
-    if use_winml:
+    if enable_training_on_device:
+        if use_cuda:
+            package_name = '/p:OrtPackageId="Microsoft.ML.OnnxRuntime.Training.Gpu"'
+        else:
+            package_name = '/p:OrtPackageId="Microsoft.ML.OnnxRuntime.Training"'
+    elif use_winml:
         package_name = '/p:OrtPackageId="Microsoft.AI.MachineLearning"'
         target_name = "/t:CreateWindowsAIPackage"
     elif use_openvino:
@@ -2210,7 +2225,7 @@ def build_nuget_package(
             run_subprocess(cmd_args, cwd=csharp_build_dir)
 
         if is_windows():
-            if use_openvino or use_tvm:
+            if not use_winml:
                 # user needs to make sure nuget is installed and added to the path variable
                 nuget_exe = "nuget.exe"
             else:
@@ -2237,12 +2252,11 @@ def build_nuget_package(
         run_subprocess(cmd_args, cwd=csharp_build_dir)
 
 
-def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt, use_dnnl):
+def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt, use_dnnl, enable_training_on_device):
     # Currently only running tests on windows.
     if not is_windows():
         return
     csharp_source_dir = os.path.join(source_dir, "csharp")
-    is_linux_build = derive_linux_build_property()
 
     # define macros based on build args
     macros = ""
@@ -2254,6 +2268,8 @@ def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt
         macros += "USE_DNNL;"
     if use_cuda:
         macros += "USE_CUDA;"
+    if enable_training_on_device:
+        macros += "__TRAINING_ENABLED_NATIVE_BUILD__;"
 
     define_constants = ""
     if macros != "":
@@ -2268,10 +2284,9 @@ def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt
     cmd_args = [
         "dotnet",
         "test",
-        "test\\Microsoft.ML.OnnxRuntime.Tests\\Microsoft.ML.OnnxRuntime.Tests.csproj",
+        "test\\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj",
         "--filter",
         "FullyQualifiedName!=Microsoft.ML.OnnxRuntime.Tests.InferenceTest.TestPreTrainedModels",
-        is_linux_build,
         define_constants,
         ort_build_dir,
     ]
@@ -2811,10 +2826,19 @@ def main():
                 args.use_tvm,
                 args.use_winml,
                 args.use_snpe,
+                args.enable_training_on_device,
             )
 
     if args.test and args.build_nuget:
-        run_csharp_tests(source_dir, build_dir, args.use_cuda, args.use_openvino, args.use_tensorrt, args.use_dnnl)
+        run_csharp_tests(
+            source_dir,
+            build_dir,
+            args.use_cuda,
+            args.use_openvino,
+            args.use_tensorrt,
+            args.use_dnnl,
+            args.enable_training_on_device,
+        )
 
     if args.gen_doc:
         # special case CI where we create the build config separately to building
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
index 932951c9f7..72d196bb38 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
@@ -192,7 +192,7 @@ jobs:
       solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
       configuration: '${{ parameters.BuildConfig }}'
       platform: 'Any CPU'
-      msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }}'
+      msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
       workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
   # C# test isn't launched by build.py, so models link has to be added.
@@ -208,7 +208,7 @@ jobs:
           command: test
           projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj'
           configuration: '${{ parameters.BuildConfig }}'
-          arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }} --blame'
+          arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) --blame'
           workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
   - ${{ if eq(parameters.EnablePython, true) }}:
diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py
index c49cfa5723..532ed8d1ef 100644
--- a/tools/nuget/generate_nuspec_for_native_nuget.py
+++ b/tools/nuget/generate_nuspec_for_native_nuget.py
@@ -306,6 +306,10 @@ def generate_files(line_list, args):
     is_dml_package = args.package_name == "Microsoft.ML.OnnxRuntime.DirectML"
     is_windowsai_package = args.package_name == "Microsoft.AI.MachineLearning"
     is_snpe_package = args.package_name == "Microsoft.ML.OnnxRuntime.Snpe"
+    is_training_package = args.package_name in [
+        "Microsoft.ML.OnnxRuntime.Training",
+        "Microsoft.ML.OnnxRuntime.Training.Gpu",
+    ]
 
     includes_winml = is_windowsai_package
     includes_directml = (is_dml_package or is_windowsai_package) and (
@@ -380,6 +384,16 @@ def generate_files(line_list, args):
         + '" target="build\\native\\include" />'
     )
 
+    if is_training_package:
+        files_list.append(
+            "<file src="
+            + '"'
+            + os.path.join(
+                args.sources_path, "orttraining\\orttraining\\training_api\\include\\onnxruntime_training_c_api.h"
+            )
+            + '" target="build\\native\\include" />'
+        )
+
     if args.execution_provider == "tvm":
         files_list.append(
             "<file src="