Add end-to-end test to run on the nuget package (#252)

* added end-to-end nuget package test

* reset the changes in OnnxRuntime.CSharp.proj

* revert the testdata directory path

* revert inference tests proj file

* added script for running end-to-end tests

* fix in the runtest.bat

* added error checking in runtest

* fixed paths in the test project

* added runtest.sh

* fix protoc path

* updated executable attributes for the runtest scripts

* added some log to debug protoc failures

* removed the protoc and duplicate test code, reuse unit-test code for end-to-end test

* copy always

* fix working dir paths in runtest.sh

* added a build.py flag to download test data without running the c++ tests

* added a script for running the test under docker

* added script for docker run of the test

csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj

@@ -32,15 +32,15 @@
   <!--TODO: Find a way to bundle the native symbol files properly -->
   <ItemGroup>
     <None Include="$(OnnxRuntimeCsharpRoot)\..\include\**\*.*"
-          PackagePath="\build\native"
+          PackagePath="\build\native\include"
           Pack="true"
-          CopyToOutputDirectory="Always"
+          CopyToOutputDirectory="Never"
           Visible="false"
     />
     <None Include="$(NativeBuildOutputDir)\onnxruntime.lib"
           PackagePath="\runtimes\win10-x64\native"
           Pack="true"
-          CopyToOutputDirectory="Always"
+          CopyToOutputDirectory="Never"
           Visible="false"
     />
     <None Include="$(NativeBuildOutputDir)\onnxruntime.dll"

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj

@@ -0,0 +1,44 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <OnnxRuntimeCsharpRoot>$(MSBuildThisFileDirectory)..\..</OnnxRuntimeCsharpRoot>
+    <Platform>AnyCPU</Platform>
+    <OutputPath>bin\$(Configuration)\</OutputPath>
+    <buildDirectory Condition="'$(buildDirectory)'==''">$(OnnxRuntimeCsharpRoot)\..\build\Windows</buildDirectory>
+    <ProtocDirectory>$(buildDirectory)\$(Configuration)\external\protobuf\cmake\$(Configuration)</ProtocDirectory>
+    <ProtoSrc>$(OnnxRuntimeCsharpRoot)\..\cmake\external\onnx\onnx</ProtoSrc>
+    <NativeBuildOutputDir>$(buildDirectory)\$(Configuration)\$(Configuration)</NativeBuildOutputDir>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.8.0" />
+    <PackageReference Include="xunit" Version="2.4.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="$(CurrentOnnxRuntimeVersion)"/>
+  </ItemGroup>
+
+
+  <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)..\Microsoft.ML.OnnxRuntime.Tests\InferenceTest.cs"/>
+    <Compile Include="$(MSBuildThisFileDirectory)..\Microsoft.ML.OnnxRuntime.Tests\OnnxMl.cs"/>
+
+    <None Include="$(OnnxRuntimeCSharpRoot)\testdata\*">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <Visible>false</Visible>
+    </None>
+    
+    <None Include="$(buildDirectory)\models\**\*">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        <Visible>false</Visible>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Service Include="{508349b6-6b84-4df5-91f0-309beebad82d}" />
+  </ItemGroup>
+
+
+</Project>

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# build docker image for CPU
+
+#TODO: Get this working, not tested yet
+
+SOURCE_ROOT=$1
+NUGET_REPO_DIRNAME=$2   # path relative to BUILD_DIR
+IMAGE="ubuntu16.04"
+PYTHON_VER=3.5
+OldDir=$(pwd)
+cd $SOURCE_ROOT/tools/ci_build/github/linux/docker
+docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .
+
+
+docker rm -f "onnxruntime-cpu" || true
+docker run -h $HOSTNAME \
+        --rm \
+        --name "onnxruntime-cpu" \
+        --volume "$SOURCE_ROOT:/onnxruntime_src" \
+        --volume "$BUILD_DIR:/home/onnxruntimedev" \
+        --volume "$HOME/.cache/onnxruntime:/root/.cache/onnxruntime" \
+        "onnxruntime-$IMAGE" \
+        /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
+        /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src &
+
+cd $OldDir

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat

@@ -0,0 +1,29 @@
+REM Copyright (c) Microsoft Corporation. All rights reserved.
+REM Licensed under the MIT License.
+
+@echo off
+SETLOCAL EnableDelayedExpansion
+
+set LocalNuGetRepo=%1
+REM WorkingDirectory is Build.SourcesDirectory\csharp
+set /p MajorVersionNumber=<..\VERSION_NUMBER
+set VersionSuffix=
+IF NOT DEFINED IsReleaseBuild (
+    FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse --short HEAD`) DO ( 
+        set VersionSuffix=-dev-%%F 
+    )
+)
+
+set CurrentOnnxRuntimeVersion=%MajorVersionNumber%%VersionSuffix%
+@echo %CurrentOnnxRuntimeVersion%
+dotnet restore test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj -s %LocalNuGetRepo% --configfile .\Nuget.CSharp.config
+if NOT errorlevel 0 (
+    @echo "Failed to restore nuget packages for the test project"
+    Exit 1
+)
+
+dotnet test test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore
+if NOT errorlevel 0 (
+    @echo "Failed to build or execute the end-to-end test"
+    Exit 1
+)

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+LocalNuGetRepo=$1
+SourceRoot=$2
+
+@echo "Downloading test data"
+python build.by --update --download_test_data
+
+MajorVersion=$(cat $SourceRoot/VERSION_NUMBER)
+VersionSuffix=
+if [ "$IsReleaseBuild" != "true" ]; then
+    VersionSuffix = -dev-$(git rev-parse --short HEAD)
+fi
+export CurrentOnnxRuntimeVersion=$MajorVersion$VersionSuffix
+echo "Current NuGet package version is $CurrentOnnxRuntimeVersion"
+
+dotnet restore $SourceRoot/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj -s $LocalNuGetRepo --configfile $SourceRoot/csharp/Nuget.CSharp.config
+if [ $? -ne 0 ]
+    echo "Failed to restore nuget packages for the test project"
+    exit 1
+)
+
+dotnet test $SourceRoot/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore
+if [ $? -ne 0 ]
+    echo "Failed to build or execute the end-to-end test"
+    exit 1
+)

tools/ci_build/build.py

@@ -58,6 +58,8 @@ Use the individual flags to only run the specified stages.
     parser.add_argument("--enable_onnx_tests", action='store_true',
                         help='''When running the Test phase, run onnx_test_running against available test data directories.''')
     parser.add_argument("--pb_home", help="Path to protobuf installation")
+    parser.add_argument("--download_test_data", action="store_true", 
+                        help='''Downloads test data without running the tests''')
     # CUDA related
     parser.add_argument("--use_cuda", action='store_true', help="Enable CUDA.")
     parser.add_argument("--cuda_home", help="Path to CUDA home."
@@ -237,7 +239,7 @@ def download_test_data(build_dir, src_url, expected_md5, azure_sas_key):
 
 def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, configs, cmake_extra_defines, args, cmake_extra_args):
     has_test_data = False
-    if args.enable_onnx_tests:
+    if args.enable_onnx_tests or args.download_test_data:
       has_test_data = download_test_data(build_dir, test_data_url, test_data_checksum, args.azure_sas_key)
     #create a shortcut for test models if there is a 'models' folder in build_dir
     if has_test_data and is_windows():