diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj index c966393f44..88b1e6ce9e 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -200,14 +200,17 @@ + $([System.DateTime]::UtcNow.ToString(yyyyMMdd)) + $([System.DateTime]::UtcNow.ToString(hhmm)) @(MajorVersionNumber) $(PackageVersion) - $(PackageVersion)-dev-$(GitCommitHashShort) + $(PackageVersion)-dev-$(CurrentDate)-$(CurrentTime)-$(GitCommitHashShort) + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest-gpu.bat b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest-gpu.bat index 2fd2dd3af3..4e9eb77561 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest-gpu.bat +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest-gpu.bat @@ -3,19 +3,9 @@ REM Licensed under the MIT License. echo on set LocalNuGetRepo=%1 +SET CurrentOnnxRuntimeVersion=%2 setlocal enableextensions disabledelayedexpansion -REM WorkingDirectory is Build.SourcesDirectory\csharp -set /p MajorVersionNumber=<..\VERSION_NUMBER -set VersionSuffix= -IF NOT DEFINED IsReleaseBuild ( - FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse --short HEAD`) DO ( - set VersionSuffix=-dev-%%F - ) -) - -set CurrentOnnxRuntimeVersion=%MajorVersionNumber%%VersionSuffix% - @echo %CurrentOnnxRuntimeVersion% pushd test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest.bat b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest.bat index 8a82907c36..e889bd3adb 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest.bat +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/runtest.bat @@ -5,19 +5,10 @@ ECHO on SET LocalNuGetRepo=%1 SET TargetArch=x64 IF NOT "%2"=="" (SET TargetArch=%2) +SET CurrentOnnxRuntimeVersion=%3 SETLOCAL enableextensions disabledelayedexpansion -REM WorkingDirectory is Build.SourcesDirectory\csharp -SET /p MajorVersionNumber=<..\VERSION_NUMBER -SET VersionSuffix= -IF NOT DEFINED IsReleaseBuild ( - FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse --short HEAD`) DO ( - SET VersionSuffix=-dev-%%F - ) -) - -SET CurrentOnnxRuntimeVersion=%MajorVersionNumber%%VersionSuffix% @ECHO %CurrentOnnxRuntimeVersion% PUSHD test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh index ab537c300f..c014732313 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh @@ -9,6 +9,7 @@ set -x SOURCE_ROOT=$1 BUILD_DIR=$2 NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR +CurrentOnnxRuntimeVersion=$4 PackageName=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu} RunTestCsharp=${RunTestCsharp:-true} RunTestNative=${RunTestNative:-true} @@ -45,7 +46,7 @@ docker run -h $HOSTNAME \ -e "RunTestNative=$RunTestNative" \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev & + /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion & wait -n diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh index a17f5b5db5..b71c51ea3e 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh @@ -9,7 +9,8 @@ set -x SOURCE_ROOT=$1 BUILD_DIR=$2 NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR -Arch=${4:-x64} # x32, x64 +CurrentOnnxRuntimeVersion=$4 +Arch=${5:-x64} # x32, x64 PackageName=${PACKAGENAME:-Microsoft.ML.OnnxRuntime} RunTestCsharp=${RunTestCsharp:-true} RunTestNative=${RunTestNative:-true} @@ -43,7 +44,7 @@ docker run -h $HOSTNAME \ -e "RunTestNative=$RunTestNative" \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev & + /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion & wait -n diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat index 08c335113e..93a863b985 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat @@ -11,6 +11,11 @@ SET dn="C:\Program Files\dotnet\dotnet" SET LocalNuGetRepo=%1 IF NOT "%2"=="" (SET TargetFramework=%2) IF NOT "%3"=="" (SET TargetArch=%3) +IF NOT "%4"=="" ( + SET CurrentOnnxRuntimeVersion=%4 +) ELSE ( + echo "Usage: runtest.bat LocalNuGetRepoPath TargetFramework TargetArch NuGetPackageVersion" +) IF "%TargetArch%"=="x86" ( SET dn="C:\Program Files (x86)\dotnet\dotnet" @@ -19,16 +24,6 @@ IF "%TargetArch%"=="x86" ( ) ECHO Target Framework is %TargetFramework% -REM WorkingDirectory is Build.SourcesDirectory\csharp -SET /p MajorVersionNumber=<..\VERSION_NUMBER -SET VersionSuffix= -IF NOT DEFINED IsReleaseBuild ( - FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse --short HEAD`) DO ( - set VersionSuffix=-dev-%%F - ) -) - -SET CurrentOnnxRuntimeVersion=%MajorVersionNumber%%VersionSuffix% REM Update if CUDA lib paths if set SET PATH=%CUDA_PATH%\bin;%CUDNN_PATH%\bin;%PATH% diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh index 6b209a166f..f1c116f5d5 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh @@ -5,28 +5,12 @@ LocalNuGetRepo=$1 SourceRoot=$2 BuildDir=$3 -#TestDataUrl=$4 -#TestDataChecksum=$5 - -#TestDataUrl and TestDataChecksum comes from the build env variable - -#echo "Downloading test data" -#python3 $SourceRoot/tools/ci_build/build.py --update --download_test_data --build_dir $BuildDir --test_data_url $TestDataUrl --test_data_checksum $TestDataChecksum -#if [ $? -ne 0 ]; then -# echo "Failed to download test data" -# exit 1 -#fi +export CurrentOnnxRuntimeVersion=$4 set -x OldDir=`pwd` cd $SourceRoot -MajorVersion=$(cat $SourceRoot/VERSION_NUMBER) -VersionSuffix= -if [ "$IsReleaseBuild" != "true" ]; then - VersionSuffix=-dev-$(git rev-parse --short=8 HEAD) -fi -export CurrentOnnxRuntimeVersion=$MajorVersion$VersionSuffix echo "Current NuGet package version is $CurrentOnnxRuntimeVersion" if [ $RunTestCsharp = "true" ]; then diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index df3d55f37d..4693e5e931 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -127,7 +127,6 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' - - template: test_all_os.yml - job: Publish_NuGet_Package_And_Report @@ -148,11 +147,15 @@ jobs: steps: - template: ../../templates/set-version-number-variables-step.yml + - template: upload-binary-sizes-from-nuget-package.yml parameters: downloadPath: $(Build.BinariesDirectory)/nuget-artifact/final-package gitCommitHash: $(OnnxRuntimeGitCommitHashShort) + - template: get-nuget-package-version-as-variable.yml + parameters: + packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - task: PowerShell@2 displayName: 'Get Current Date' @@ -166,7 +169,7 @@ jobs: displayName: 'Copy Signed NuGet Package to Blob Store' condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme inputs: - sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + sourcePath: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(NuGetPackageVersionNumber).nupkg' azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' destination: azureBlob storage: ortpackages @@ -178,5 +181,5 @@ jobs: condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme inputs: command: 'push' - searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(OnnxRuntimeVersion)-dev-$(OnnxRuntimeGitCommitHashShort).nupkg' + searchPatternPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(NuGetPackageVersionNumber).nupkg' feedPublish: 'OnnxRuntime' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/get-nuget-package-version-as-variable.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/get-nuget-package-version-as-variable.yml new file mode 100644 index 0000000000..9114a4f2d5 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/get-nuget-package-version-as-variable.yml @@ -0,0 +1,28 @@ +parameters: + packageFolder: $(Build.ArtifactStagingDirectory) + +steps: +- task: CmdLine@2 + condition: eq(variables['Agent.OS'], 'Windows_NT') + displayName: 'Extract version number from the NuPkg file, Windows VMs' + inputs: + workingDirectory: '${{ parameters.packageFolder }}' + script: | + SETLOCAL EnableDelayedExpansion + FOR /R %%i IN (Microsoft.ML.OnnxRuntime*.nupkg) do ( + set filename=%%~ni + set ortversion=!filename:~25! + @echo ortversion is !ortversion! + @echo ##vso[task.setvariable variable=NuGetPackageVersionNumber;]!ortversion! + ) + +- task: CmdLine@2 + condition: not(eq(variables['Agent.OS'], 'Windows_NT')) + displayName: 'Extract version number from the NuPkg file, Unix VMs' + inputs: + workingDirectory: '${{ parameters.packageFolder }}' + script: | + filenamewithext=$(ls Microsoft.ML.OnnxRuntime*.nupkg) + filename=${filenamewithext%.*} + ortversion=${filename:25} + echo "##vso[task.setvariable variable=NuGetPackageVersionNumber;]$ortversion" diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 2cb8763745..63891a2eea 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -19,10 +19,14 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + - template: get-nuget-package-version-as-variable.yml + parameters: + packageFolder: '$(Build.BinariesDirectory)/nuget-artifact' + - ${{ if eq(parameters['TestGPU'], 'false') }}: - script: | set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact + $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact $(NuGetPackageVersionNumber) displayName: 'Run Package Test (x64)' env: OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) @@ -30,7 +34,7 @@ jobs: - ${{ if eq(parameters['TestGPU'], 'true') }}: - script: | set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact + $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact $(NuGetPackageVersionNumber) displayName: 'Run Package Test GPU (x64)' env: OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index 03bb2cf419..eef996e9db 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -18,11 +18,16 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + - template: get-nuget-package-version-as-variable.yml + parameters: + packageFolder: '$(Build.BinariesDirectory)/nuget-artifact' + - script: | $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ $(Build.BinariesDirectory)/nuget-artifact \ $(Build.SourcesDirectory) \ - $(Build.BinariesDirectory) + $(Build.BinariesDirectory) \ + $(NuGetPackageVersionNumber) if [ $? -ne 0 ]; then echo "Failed to run test" diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index fb04583779..0615dd9b6f 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -21,24 +21,28 @@ jobs: displayName: 'Download Pipeline Artifact' inputs: artifactName: 'drop-signed-nuget' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + targetPath: '$(Build.BinariesDirectory)\nuget-artifact' + - template: get-nuget-package-version-as-variable.yml + parameters: + packageFolder: '$(Build.BinariesDirectory)\nuget-artifact' + - script: | - @echo "Running build.py --update" - $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --config Debug --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --update --download_test_data --test_data_url=$(TestDataUrl) --test_data_checksum=$(TestDataChecksum) + @echo "Running download_test_data.py" + $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) displayName: 'Download Test Data' - script: | @echo "Running Runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact + test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact netcoreapp2.1 x64 $(NuGetPackageVersionNumber) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Run End to End Test (C#) .Net Core' continueOnError: true - script: | @echo "Running Runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact netcoreapp2.1 x86 + test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact netcoreapp2.1 x86 $(NuGetPackageVersionNumber) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Run End to End Test (C#) .Net Core x86' enabled: false @@ -46,14 +50,14 @@ jobs: - script: | @echo "Running Runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact net461 + test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact net461 x64 $(NuGetPackageVersionNumber) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Run End to End Test (C#) .NetFramework' continueOnError: true - script: | @echo "Running Runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact net461 x86 + test\Microsoft.ML.OnnxRuntime.EndToEndTests\runtest.bat $(Build.BinariesDirectory)\nuget-artifact net461 x86 $(NuGetPackageVersionNumber) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Run End to End Test (C#) .NetFramework x86' enabled: false @@ -61,7 +65,7 @@ jobs: - script: | @echo "Running runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi\runtest.bat $(Build.BinariesDirectory)\nuget-artifact + test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi\runtest.bat $(Build.BinariesDirectory)\nuget-artifact x64 $(NuGetPackageVersionNumber) REM Need an ErrorLevel check, since the script uses Exit /B IF NOT %ERRORLEVEL% EQU 0 { EXIT 1 @@ -77,7 +81,7 @@ jobs: - script: | @echo "Running runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi\runtest.bat $(Build.BinariesDirectory)\nuget-artifact + test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi\runtest.bat $(Build.BinariesDirectory)\nuget-artifact x64 $(NuGetPackageVersionNumber) REM Need an ErrorLevel check, since the script uses Exit /B IF NOT %ERRORLEVEL% EQU 0 { EXIT 1 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml index 6f89951fa1..c82cb4b085 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml @@ -24,6 +24,7 @@ steps: - task: CmdLine@2 displayName: 'Post binary sizes to the dashboard database using command line' + continueOnError: true # Recently the database connection is failing, so ignoring the error until that is fixed inputs: script: | echo changing directory to artifact download path @@ -33,7 +34,7 @@ steps: echo processing %%~ni.nupkg copy %%~ni.nupkg %%~ni.zip echo copied to zip - echo listing lib files in the zip + echo listing lib files in the zip REM use a single .csv file to put the data echo os,arch,build_config,size > binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,openmp,%%a >> binary_size_data.txt