From 65201e47bf58ce5bbc8389a83082f69ee1ea50b3 Mon Sep 17 00:00:00 2001
From: Ashwini Khade <askhade@microsoft.com>
Date: Mon, 5 Dec 2022 14:54:09 -0800
Subject: [PATCH] Enable nuget packages for on device training (#13637)

### Description
This PR enables building nuget packages locally for on device training
using --build_nuget arg.
This PR also enables the C# bindings by default in the managed package.
If a user triggers any training apis when the native binary is not built
for training, an exception with message "Training is disabled in the
current build. Please build ONNXRuntime from source with the build flags
enable_training and enable_training_on_device. " is thrown.

Build command for creating nuget packes for on device training:
build.bat --enable_training --enable_training_on_device --build_nuget

2 Nuget packages are built
1. Microsoft.ML.OnnxRuntime.Managed
2. Microsoft.ML.OnnxRuntime.Training OR
Microsoft.ML.OnnxRuntime.Training.Gpu



### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
---
 cmake/onnxruntime_csharp.cmake                |  4 ++
 csharp/OnnxRuntime.CSharp.proj                | 16 ++---
 .../Microsoft.ML.OnnxRuntime.csproj           | 68 ++++++++++---------
 .../Training/CheckpointState.shared.cs        |  2 +-
 .../Training/TrainingSession.shared.cs        |  2 +-
 .../TrainingTest.cs                           | 18 ++++-
 ...oft.ML.OnnxRuntime.Tests.NetCoreApp.csproj | 20 ++++--
 tools/ci_build/build.py                       | 40 ++++++++---
 .../templates/win-ci-vs-2019.yml              |  4 +-
 .../nuget/generate_nuspec_for_native_nuget.py | 14 ++++
 10 files changed, 127 insertions(+), 61 deletions(-)

diff --git a/cmake/onnxruntime_csharp.cmake b/cmake/onnxruntime_csharp.cmake
index 58a9f894ec..c65acd849f 100644
--- a/cmake/onnxruntime_csharp.cmake
+++ b/cmake/onnxruntime_csharp.cmake
@@ -50,6 +50,10 @@ if (onnxruntime_USE_XNNPACK)
   STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_XNNPACK;")
 endif()
 
+if (onnxruntime_ENABLE_TRAINING_ON_DEVICE)
+  STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "__TRAINING_ENABLED_NATIVE_BUILD__;")
+endif()
+
 include(CSharpUtilities)
 
 # generate Directory.Build.props
diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj
index 94635f2dc3..5473246e8b 100644
--- a/csharp/OnnxRuntime.CSharp.proj
+++ b/csharp/OnnxRuntime.CSharp.proj
@@ -1,12 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!--
-This is the master msbuild project file for all csharp components. 
+This is the master msbuild project file for all csharp components.
 This is created so that the NuGet dependencies are restored before the projects are built during a CI build.
 CMake creates a target to this project
 -->
 
 <Project DefaultTargets="Build">
-  
+
   <PropertyGroup>
     <OrtPackageId Condition=" '$(OrtPackageId)' == '' ">Microsoft.ML.OnnxRuntime</OrtPackageId>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -54,7 +54,7 @@ CMake creates a target to this project
    <Target Name="ObtainPackageVersion" BeforeTargets="Build;CreatePackage;CreateWindowsAIPackage">
     <ReadLinesFromFile File="..\VERSION_NUMBER">
       <Output TaskParameter="Lines" ItemName="MajorVersionNumber"/>
-    </ReadLinesFromFile>   
+    </ReadLinesFromFile>
     <Exec Command="git rev-parse --short HEAD" ConsoleToMSBuild="true">
         <Output TaskParameter="ConsoleOutput" PropertyName="GitCommitHashShort" />
     </Exec>
@@ -72,7 +72,7 @@ CMake creates a target to this project
     </ConvertToAbsolutePath>
     <ConvertToAbsolutePath Paths="$(OnnxRuntimeSourceDirectory)">
       <Output TaskParameter="AbsolutePaths" PropertyName="OnnxRuntimeSourceDirectoryAbs"/>
-    </ConvertToAbsolutePath>    
+    </ConvertToAbsolutePath>
     <PropertyGroup>
       <CurrentDate>$([System.DateTime]::UtcNow.ToString(yyyyMMdd))</CurrentDate>
       <CurrentTime>$([System.DateTime]::UtcNow.ToString(hhmm))</CurrentTime>
@@ -82,12 +82,12 @@ CMake creates a target to this project
     </PropertyGroup>
     <Message Importance="High" Text="PackageVersion=$(PackageVersion)" />
   </Target>
-  
+
   <Target Name="CreatePackage">
     <Message Importance="High" Text="Bundling managed assemblies into a NuGet package ..." />
     <MSBuild Projects="src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
              Targets="CopyMiscFiles;Pack"
-            Properties="NoBuild=true;Platform=AnyCPU;PackageVersion=$(PackageVersion);OrtPackageId=$(OrtPackageId);SelectedTargets=All"/> 
+            Properties="NoBuild=true;Platform=AnyCPU;PackageVersion=$(PackageVersion);OrtPackageId=$(OrtPackageId);SelectedTargets=All"/>
 
     <Message Importance="High" Text="Generating nuspec for the native Nuget package ..." />
     <Exec ContinueOnError="False" Command="python $(GenerateNuspecScript) --package_version $(PackageVersion) --package_name $(OrtPackageId) --target_architecture $(TargetArchitecture) --build_config $(Configuration) --native_build_path $(NativeBuildOutputDirAbs) --packages_path $(OnnxRuntimePackagesDirectoryAbs) --ort_build_path $(OnnxRuntimeBuildDirectoryAbs) --sources_path $(OnnxRuntimeSourceDirectoryAbs) --commit_id $(GitCommitHash) --is_release_build $(IsReleaseBuild) --execution_provider $(ExecutionProvider)" ConsoleToMSBuild="true">
@@ -105,7 +105,7 @@ CMake creates a target to this project
 
     <Copy
     SourceFiles="$(NativeBuildOutputDirAbs)\$(OrtPackageId).$(PackageVersion).nupkg"
-    DestinationFolder="$(NativeBuildOutputDirAbs)\nuget-artifacts"
+    DestinationFolder="$(NativeBuildOutputDirAbs)\nuget-local-artifacts"
     />
   </Target>
 
@@ -113,7 +113,7 @@ CMake creates a target to this project
     <LicenseFile Include="$(OnnxRuntimeSourceDirectory)\LICENSE"/>
   </ItemGroup>
 
-  <Target Name="CreateWindowsAIPackage">  
+  <Target Name="CreateWindowsAIPackage">
     <Copy SourceFiles="@(LicenseFile)" DestinationFiles="@(LicenseFile->'$(OnnxRuntimeSourceDirectory)\%(Filename).txt')"/>
 
     <Message Importance="High" Text="Generating nuspec for the Microsoft.AI.MachineLearning Nuget package ..." />
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
index 679361c336..e6a8272fd8 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
@@ -4,19 +4,19 @@
     <OrtPackageId Condition="'$(OrtPackageId)' == ''">Microsoft.ML.OnnxRuntime</OrtPackageId>
   </PropertyGroup>
 
-  <!-- 
+  <!--
   Temporary setup until official Visual Studio 2022 release supports .net6, as the CIs require the official release.
   We will be able to build all targets with VS once that happens.
- 
+
   Until then, we need to build the pre-.net6 targets with VS and the .net6 targets with dotnet.
-  
+
   The pre-.net6 Xamarin targets are optional and only included if the machine has the required workloads.
 
   We have 3 scenarios
     1) Build pre-net6 targets with VS - SelectedTargets=PreNet6
     2) Build net6 targets - SelectedTargets=Net6
     3) Run 'Pack' target to create nuget package from combination of 1 and 2 - SelectedTargets=All
-  
+
   Default is PreNet6 so that existing projects and CI builds will do the same thing unless explicitly updated.
   -->
   <PropertyGroup>
@@ -24,18 +24,18 @@
     <BaseTargets>netstandard1.1;netstandard2.0;net5.0;netcoreapp3.1</BaseTargets>
   </PropertyGroup>
 
-  <!-- only set the Xamarin mobile targets if we're building an ORT package, 
+  <!-- only set the Xamarin mobile targets if we're building an ORT package,
        and only if the mobile workloads are installed -->
-  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR 
-                             '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu') AND 
-                             Exists('$(MSBuildExtensionsPath)\Xamarin\Android') AND 
+  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR
+                             '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu') AND
+                             Exists('$(MSBuildExtensionsPath)\Xamarin\Android') AND
                              Exists('$(MSBuildExtensionsPath)\Xamarin\iOS')">
     <XamarinTargets>xamarinios10;monoandroid11.0</XamarinTargets>
   </PropertyGroup>
 
   <!-- only set the .net6 targets if we're building an ORT package.
        we can add .net6 support to other packages later as needed -->
-  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR 
+  <PropertyGroup Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR
                              '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu')">
     <Net6Targets>net6.0;net6.0-android;net6.0-ios;net6.0-macos</Net6Targets>
   </PropertyGroup>
@@ -64,7 +64,7 @@
     <!--internal build related properties-->
     <OnnxRuntimeRoot>$(ProjectDir)..\..\..</OnnxRuntimeRoot>
     <OnnxRuntimeCsharpRoot>$(OnnxRuntimeRoot)\csharp</OnnxRuntimeCsharpRoot>
-    <TargetArchitecture Condition=" '$(TargetArchitecture)' == '' ">x64</TargetArchitecture>  
+    <TargetArchitecture Condition=" '$(TargetArchitecture)' == '' ">x64</TargetArchitecture>
 
     <RootNamespace>Microsoft.ML.OnnxRuntime</RootNamespace>
     <AssemblyName>Microsoft.ML.OnnxRuntime</AssemblyName>
@@ -72,11 +72,15 @@
     <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
     <DebugType>portable</DebugType>
 
-    <!-- Training build property. With Visual Studio, change to true to enable. -->
-    <OnDeviceTraining Condition="'$(OnDeviceTraining)' == ''">false</OnDeviceTraining>
+    <!-- Controls whether C# Bindings for Training are included in the Managed Nuget Package.
+    Bindings for training are included by default. If user tries to call training apis when the native package installed
+    on their device is not built for training, an exception will be thrown with the following message -
+    "Training is disabled in the current build. Please build onnxruntime from source with the build flags
+    enable_training and enable_training_on_device. "-->
+   <OnDeviceTraining Condition="'$(OnDeviceTraining)' == ''">true</OnDeviceTraining>
 
     <!--- The package name is always hardcoded as the package created by this project only contains managed assemblies -->
-    <!--- The parameter OrtPackageId is only used for some conditional logic below -->    
+    <!--- The parameter OrtPackageId is only used for some conditional logic below -->
     <PackageId>Microsoft.ML.OnnxRuntime.Managed</PackageId>
     <Authors>Microsoft</Authors>
     <PackageVersion Condition=" '$(PackageVersion)' == '' And '$(Configuration)' == 'Debug' ">1.0.0</PackageVersion>
@@ -105,27 +109,35 @@
     <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
     <Configurations>Debug;Release;RelWithDebInfo</Configurations>
 
-    <IsLinuxBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinuxBuild> 
-    <IsWindowsBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindowsBuild> 
+    <IsLinuxBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinuxBuild>
+    <IsWindowsBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindowsBuild>
     <IsMacOSBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsMacOSBuild>
 
-    <!-- $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) gives better results than 
+    <!-- $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) gives better results than
          $(TargetPlatformIdentifier). See https://github.com/dotnet/msbuild/issues/7359
          Note there are slight differences in casing (e.g. macos vs macOS), so if we ever
          change to use $(TargetPlatformIdentifier) we need to adjust for that.
-         
+
     -->
-    <IsXamarinTarget Condition="$(TargetFramework.StartsWith('xamarinios')) OR 
+    <IsXamarinTarget Condition="$(TargetFramework.StartsWith('xamarinios')) OR
                                 $(TargetFramework.StartsWith('monoandroid'))">true</IsXamarinTarget>
-    <IsAndroidTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android' OR 
+    <IsAndroidTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android' OR
                                 $(TargetFramework.StartsWith('monoandroid'))">true</IsAndroidTarget>
-    <IsIOSTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'ios' OR 
+    <IsIOSTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'ios' OR
                             $(TargetFramework.StartsWith('xamarinios'))">true</IsIOSTarget>
     <IsMacTarget Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'macos' OR
                             $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'maccatalyst'">true</IsMacTarget>
   </PropertyGroup>
 
-  <!-- 
+  <!-- Enable training APIs for the build. The native package must be
+       a training build with build flags 'enable_training' and
+       'enable_training_on_device' set.
+  -->
+  <PropertyGroup Condition="'$(OnDeviceTraining)'=='true'">
+    <OrtConstants>$(OrtConstants);__ENABLE_TRAINING_ON_DEVICE__</OrtConstants>
+  </PropertyGroup>
+
+  <!--
     Properties that depend on the system we're building on.
   -->
   <PropertyGroup Condition="'$(IsLinuxBuild)'=='true'">
@@ -146,7 +158,7 @@
       <NativeBuildOutputDir>$(OnnxRuntimeBuildDirectory)\$(Configuration)</NativeBuildOutputDir>
   </PropertyGroup>
 
-  <!-- 
+  <!--
     Properties that depend on the target framework.
   -->
   <PropertyGroup Condition="'$(IsIOSTarget)'=='true' OR '$(IsAndroidTarget)'=='true'">
@@ -179,14 +191,6 @@
     <OrtConstants>$(OrtConstants);__XAMARIN__</OrtConstants>
   </PropertyGroup>
 
-  <!-- Enable training APIs for the build. The native package must be
-       a training build with build flags 'enable_training' and
-       'enable_training_on_device' set.
-  -->
-  <PropertyGroup Condition="'$(OnDeviceTraining)' == 'true'">
-    <OrtConstants>$(OrtConstants);__ENABLE_TRAINING_ON_DEVICE__</OrtConstants>
-  </PropertyGroup>
-
   <PropertyGroup>
     <DefineConstants>$(DefineConstants);$(OrtConstants)</DefineConstants>
   </PropertyGroup>
@@ -287,7 +291,7 @@
     <None Include="targets\netstandard\$(PackageId).targets"
           PackagePath="build\netstandard1.1\$(PackageId).targets;build\netstandard2.0\$(PackageId).targets"
           Pack="true"
-          Visible="false" 
+          Visible="false"
     />
 
     <!-- Some tools to be packaged in nightly build only, should not be released -->
@@ -346,5 +350,5 @@
       DestinationFolder="$(NativeBuildOutputDir)"
     />
   </Target>
-  
+
 </Project>
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs
index 4cff5bccb1..fec2c18572 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs
@@ -33,7 +33,7 @@ namespace Microsoft.ML.OnnxRuntime
             }
             else
             {
-                throw new InvalidOperationException("Training is disabled in the current build");
+                throw new InvalidOperationException("Training is disabled in the current build. Please build ONNXRuntime from source with the build flags enable_training and enable_training_on_device. \n");
             }
         }
 
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs
index b5b5a7c598..8d804628af 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs
@@ -336,7 +336,7 @@ namespace Microsoft.ML.OnnxRuntime
         {
             if (!NativeTrainingMethods.TrainingEnabled())
             {
-                throw new InvalidOperationException("Training is disabled in the current build.");
+                throw new InvalidOperationException("Training is disabled in the current build. Please build ONNXRuntime from source with the build flags enable_training and enable_training_on_device. \n");
             }
             var options = sessOptions;
             if (sessOptions == null)
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs
index f5bf62b4b8..f643efa720 100644
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs
@@ -23,6 +23,17 @@ namespace Microsoft.ML.OnnxRuntime.Tests
             this.output = o;
         }
 
+#if !__TRAINING_ENABLED_NATIVE_BUILD__
+        [Fact(DisplayName = "TestLoadCheckpointThrows")]
+        public void TestLoadCheckpointThrows()
+        {
+            string path = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt");
+            var ex = Assert.Throws<InvalidOperationException>(() => { var opt = new CheckpointState(path); });
+            Assert.Contains("Training is disabled in the current build.", ex.Message);
+        }
+#endif
+
+#if __TRAINING_ENABLED_NATIVE_BUILD__
         [Fact(DisplayName = "TestLoadCheckpoint")]
         public void TestLoadCheckpoint()
         {
@@ -122,13 +133,13 @@ namespace Microsoft.ML.OnnxRuntime.Tests
                 outputs = trainingSession.TrainStep(pinnedInputs);
                 var outputBuffer = outputs.ElementAtOrDefault(0);
 
-                Assert.Equal("542.loss", outputBuffer.Name);
+                Assert.Equal("onnx::loss::21273", outputBuffer.Name);
                 Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, outputBuffer.ValueType);
                 Assert.Equal(TensorElementType.Float, outputBuffer.ElementType);
 
                 var outLabelTensor = outputBuffer.AsTensor<float>();
                 Assert.NotNull(outLabelTensor);
-                Assert.Equal(expectedOutput, outLabelTensor, new FloatComparer());
+                Assert.Equal(expectedOutput, outLabelTensor.ToArray(), new FloatComparer());
             }
         }
 
@@ -304,5 +315,6 @@ namespace Microsoft.ML.OnnxRuntime.Tests
                 return x.GetHashCode();
             }
         }
-    }
+#endif
+        }
 }
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj
index fbb0417215..dafebd9ccb 100644
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj
@@ -17,10 +17,18 @@
     <AssemblyOriginatorKeyFile>..\..\OnnxRuntime.snk</AssemblyOriginatorKeyFile>
     <Configurations>Debug;Release;RelWithDebInfo</Configurations>
     <!-- end -->
-    <DefineConstants>__NET_CORE_APP__;$(DefineConstants)</DefineConstants>
 
-    <!-- Training build property. With Visual Studio, change to true to enable. -->
-    <OnDeviceTraining Condition="'$(OnDeviceTraining)' == ''">false</OnDeviceTraining>
+    <!-- Training build property.
+    Should be set to true when training is enabled in onnxruntime native binary -->
+    <TrainingEnabledNativeBuild Condition="'$(TrainingEnabledNativeBuild)' == ''">false</TrainingEnabledNativeBuild>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TrainingEnabledNativeBuild)'=='true'">
+    <ExtraDefineConstants>$(ExtraDefineConstants);__TRAINING_ENABLED_NATIVE_BUILD__</ExtraDefineConstants>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <DefineConstants>__NET_CORE_APP__;$(DefineConstants);$(ExtraDefineConstants)</DefineConstants>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(IsLinuxBuild)'=='true'">
@@ -95,12 +103,12 @@
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\Tensors\ArrayTensorExtensionsTests.cs">
       <Link>ArrayTensorExtensionsTests.cs</Link>
     </Compile>
-  </ItemGroup>
-
-  <ItemGroup Condition="'$(OnDeviceTraining)' == 'true'">
     <Compile Include="..\Microsoft.ML.OnnxRuntime.Tests.Common\TrainingTest.cs">
       <Link>TrainingTest.cs</Link>
     </Compile>
+  </ItemGroup>
+
+  <ItemGroup>
     <None Include="$(OnnxRuntimeCSharpRoot)\..\onnxruntime\test\testdata\training_api\**\*.*">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <Visible>false</Visible>
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index 6d094c8c6a..a6e6b4cdc7 100644
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -2131,7 +2131,17 @@ def derive_linux_build_property():
 
 
 def build_nuget_package(
-    source_dir, build_dir, configs, use_cuda, use_openvino, use_tensorrt, use_dnnl, use_tvm, use_winml, use_snpe
+    source_dir,
+    build_dir,
+    configs,
+    use_cuda,
+    use_openvino,
+    use_tensorrt,
+    use_dnnl,
+    use_tvm,
+    use_winml,
+    use_snpe,
+    enable_training_on_device,
 ):
     if not (is_windows() or is_linux()):
         raise BuildError(
@@ -2150,7 +2160,12 @@ def build_nuget_package(
     target_name = "/t:CreatePackage"
     execution_provider = '/p:ExecutionProvider="None"'
     package_name = '/p:OrtPackageId="Microsoft.ML.OnnxRuntime"'
-    if use_winml:
+    if enable_training_on_device:
+        if use_cuda:
+            package_name = '/p:OrtPackageId="Microsoft.ML.OnnxRuntime.Training.Gpu"'
+        else:
+            package_name = '/p:OrtPackageId="Microsoft.ML.OnnxRuntime.Training"'
+    elif use_winml:
         package_name = '/p:OrtPackageId="Microsoft.AI.MachineLearning"'
         target_name = "/t:CreateWindowsAIPackage"
     elif use_openvino:
@@ -2210,7 +2225,7 @@ def build_nuget_package(
             run_subprocess(cmd_args, cwd=csharp_build_dir)
 
         if is_windows():
-            if use_openvino or use_tvm:
+            if not use_winml:
                 # user needs to make sure nuget is installed and added to the path variable
                 nuget_exe = "nuget.exe"
             else:
@@ -2237,12 +2252,11 @@ def build_nuget_package(
         run_subprocess(cmd_args, cwd=csharp_build_dir)
 
 
-def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt, use_dnnl):
+def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt, use_dnnl, enable_training_on_device):
     # Currently only running tests on windows.
     if not is_windows():
         return
     csharp_source_dir = os.path.join(source_dir, "csharp")
-    is_linux_build = derive_linux_build_property()
 
     # define macros based on build args
     macros = ""
@@ -2254,6 +2268,8 @@ def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt
         macros += "USE_DNNL;"
     if use_cuda:
         macros += "USE_CUDA;"
+    if enable_training_on_device:
+        macros += "__TRAINING_ENABLED_NATIVE_BUILD__;"
 
     define_constants = ""
     if macros != "":
@@ -2268,10 +2284,9 @@ def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt
     cmd_args = [
         "dotnet",
         "test",
-        "test\\Microsoft.ML.OnnxRuntime.Tests\\Microsoft.ML.OnnxRuntime.Tests.csproj",
+        "test\\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj",
         "--filter",
         "FullyQualifiedName!=Microsoft.ML.OnnxRuntime.Tests.InferenceTest.TestPreTrainedModels",
-        is_linux_build,
         define_constants,
         ort_build_dir,
     ]
@@ -2811,10 +2826,19 @@ def main():
                 args.use_tvm,
                 args.use_winml,
                 args.use_snpe,
+                args.enable_training_on_device,
             )
 
     if args.test and args.build_nuget:
-        run_csharp_tests(source_dir, build_dir, args.use_cuda, args.use_openvino, args.use_tensorrt, args.use_dnnl)
+        run_csharp_tests(
+            source_dir,
+            build_dir,
+            args.use_cuda,
+            args.use_openvino,
+            args.use_tensorrt,
+            args.use_dnnl,
+            args.enable_training_on_device,
+        )
 
     if args.gen_doc:
         # special case CI where we create the build config separately to building
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
index 932951c9f7..72d196bb38 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
@@ -192,7 +192,7 @@ jobs:
       solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
       configuration: '${{ parameters.BuildConfig }}'
       platform: 'Any CPU'
-      msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }}'
+      msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
       workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
   # C# test isn't launched by build.py, so models link has to be added.
@@ -208,7 +208,7 @@ jobs:
           command: test
           projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj'
           configuration: '${{ parameters.BuildConfig }}'
-          arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }} --blame'
+          arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) --blame'
           workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
   - ${{ if eq(parameters.EnablePython, true) }}:
diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py
index c49cfa5723..532ed8d1ef 100644
--- a/tools/nuget/generate_nuspec_for_native_nuget.py
+++ b/tools/nuget/generate_nuspec_for_native_nuget.py
@@ -306,6 +306,10 @@ def generate_files(line_list, args):
     is_dml_package = args.package_name == "Microsoft.ML.OnnxRuntime.DirectML"
     is_windowsai_package = args.package_name == "Microsoft.AI.MachineLearning"
     is_snpe_package = args.package_name == "Microsoft.ML.OnnxRuntime.Snpe"
+    is_training_package = args.package_name in [
+        "Microsoft.ML.OnnxRuntime.Training",
+        "Microsoft.ML.OnnxRuntime.Training.Gpu",
+    ]
 
     includes_winml = is_windowsai_package
     includes_directml = (is_dml_package or is_windowsai_package) and (
@@ -380,6 +384,16 @@ def generate_files(line_list, args):
         + '" target="build\\native\\include" />'
     )
 
+    if is_training_package:
+        files_list.append(
+            "<file src="
+            + '"'
+            + os.path.join(
+                args.sources_path, "orttraining\\orttraining\\training_api\\include\\onnxruntime_training_c_api.h"
+            )
+            + '" target="build\\native\\include" />'
+        )
+
     if args.execution_provider == "tvm":
         files_list.append(
             "<file src="