saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-26 22:35:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update packaging pipelines(#6664)

Browse source
This commit is contained in:
Changming Sun 2021-02-17 09:53:36 -08:00 committed by GitHub
parent 46c06f6ac7
commit 0be5475de6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 53 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
csharp/OnnxRuntime.CSharp.proj
View file

@ -114,7 +114,7 @@ CMake creates a target to this project
<CurrentTime>$([System.DateTime]::UtcNow.ToString(hhmm))</CurrentTime>
<PackageVersion>@(MajorVersionNumber)</PackageVersion>
<Version>$(PackageVersion)</Version>
<PackageVersion Condition="'$(IsReleaseBuild)'!='true'">$(PackageVersion)-dev-$(CurrentDate)-$(CurrentTime)-$(GitCommitHashShort)</PackageVersion>
<PackageVersion Condition="'$(IsReleaseBuild)'!='True'">$(PackageVersion)-dev-$(CurrentDate)-$(CurrentTime)-$(GitCommitHashShort)</PackageVersion>
</PropertyGroup>
<Message Importance="High" Text="PackageVersion=$(PackageVersion)" />
</Target>

4
csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh
View file

@ -11,7 +11,7 @@ BUILD_DIR=$2
NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR
CurrentOnnxRuntimeVersion=$4
DockerImage=$5
PackageName=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu}
PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu}
RunTestCsharp=${RunTestCsharp:-true}
RunTestNative=${RunTestNative:-true}
@ -25,7 +25,7 @@ docker run --gpus all --rm \
--volume /data/models:/home/onnxruntimedev/models:ro \
-e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \
-e "IsReleaseBuild=$ISRELEASEBUILD" \
-e "PackageName=$PackageName" \
-e "PACKAGENAME=$PACKAGENAME" \
-e "RunTestCsharp=$RunTestCsharp" \
-e "RunTestNative=$RunTestNative" \
-e "BUILD_BINARIESDIRECTORY=/home/onnxruntimedev" \

4
csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh
View file

@ -13,7 +13,7 @@ CurrentOnnxRuntimeVersion=$4
DockerImage=$5
UseCentos7=${6:-false}
Arch=${7:-x64} # x32, x64
PackageName=${PACKAGENAME:-Microsoft.ML.OnnxRuntime}
PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime}
RunTestCsharp=${RunTestCsharp:-true}
RunTestNative=${RunTestNative:-true}
PYTHON_VER=3.5
@ -31,7 +31,7 @@ docker run --rm \
--volume /data/models:/home/onnxruntimedev/models:ro \
-e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \
-e "IsReleaseBuild=$ISRELEASEBUILD" \
-e "PackageName=$PackageName" \
-e "PACKAGENAME=$PACKAGENAME" \
-e "DisableContribOps=$DISABLECONTRIBOPS" \
-e "DisableMlOps=$DISABLEMLOPS" \
-e "RunTestCsharp=$RunTestCsharp" \

9
csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh
View file

@ -6,8 +6,7 @@ LocalNuGetRepo=$1
BuildDir=$3
export CurrentOnnxRuntimeVersion=$4
IsMacOS=${5:-false}
# NOTE: PackageName is not PACKAGENAME since this is being called by other scripts that have already switched to the CamelCase version
PackageName=${PackageName:-Microsoft.ML.OnnxRuntime}
PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime}
RunTestCsharp=${RunTestCsharp:-true}
RunTestNative=${RunTestNative:-true}
@ -39,12 +38,12 @@ fi
if [ $RunTestNative = "true" ]; then
# Run Native shared object test
# PackageName is passed in environment (e.g. Microsoft.ML.OnnxRuntime)
PackageName="$PackageName.$CurrentOnnxRuntimeVersion.nupkg"
# PACKAGENAME is passed in environment (e.g. Microsoft.ML.OnnxRuntime)
PACKAGENAME="$PACKAGENAME.$CurrentOnnxRuntimeVersion.nupkg"
cd $LocalNuGetRepo
TempDir=_tmp
mkdir -p $TempDir && pushd $TempDir
unzip ../$PackageName
unzip ../$PACKAGENAME
inc="-I build/native/include"

2
tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml
View file

@ -3,7 +3,7 @@ jobs:
timeoutInMinutes: 180
workspace:
clean: all
pool: Linux-GPU-CUDA10
pool: Onnxruntime-Linux-GPU
steps:
- checkout: self
clean: true

27
tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml
View file

@ -38,7 +38,7 @@ jobs:
workspace:
clean: all
timeoutInMinutes: 120
pool: 'Linux-GPU-CUDA10'
pool: 'Onnxruntime-Linux-GPU'
variables:
CUDA_VERSION: '11.0'
steps:
@ -97,6 +97,7 @@ jobs:
buildparameter: --use_cuda --cuda_version=11.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" --cudnn_home="C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda" --enable_onnx_tests --enable_wcos --build_java
runTests: ${{ parameters.RunOnnxRuntimeTests }}
buildJava: true
java_artifact_id: onnxruntime_gpu
- job: Jar_Packaging_GPU
workspace:
@ -192,7 +193,7 @@ jobs:
popd
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar"
java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
- template: ../templates/component-governance-component-detection-steps.yml
@ -205,7 +206,7 @@ jobs:
- job: Final_Jar_Testing_Linux_GPU
workspace:
clean: all
pool: 'Linux-GPU-CUDA10'
pool: 'Onnxruntime-Linux-GPU'
variables:
- name: runCodesignValidationInjection
value: false
@ -213,6 +214,8 @@ jobs:
dependsOn:
Jar_Packaging_GPU
steps:
- checkout: self
submodules: false
- template: ../templates/set-version-number-variables-step.yml
- task: DownloadPipelineArtifact@2
displayName: 'Download Final Jar'
@ -221,20 +224,12 @@ jobs:
artifactName: 'onnxruntime-java-gpu'
targetPath: '$(Build.BinariesDirectory)/final-jar'
- task: CmdLine@2
- task: Bash@3
displayName: 'Test'
inputs:
script: |
echo "Java Version"
java -version
mkdir test
pushd test
jar xf $(Build.BinariesDirectory)/final-jar/testing.jar
popd
wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./
wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./
LD_LIBRARY_PATH=./test:${LD_LIBRARY_PATH}
java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
workingDirectory: '$(Build.BinariesDirectory)/final-jar'
targetType: filePath
filePath: 'tools/ci_build/github/linux/java_linux_final_test.sh'
arguments: '-r $(Build.BinariesDirectory) -v $(OnnxRuntimeVersion)'
- template: ../templates/component-governance-component-detection-steps.yml
parameters :

4
tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
View file

@ -75,7 +75,7 @@ jobs:
workspace:
clean: all
timeoutInMinutes: 120
pool: 'Linux-GPU-CUDA10'
pool: 'Onnxruntime-Linux-GPU'
steps:
- template: ../../templates/set-version-number-variables-step.yml
- template: ../../templates/get-docker-image-steps.yml
@ -237,7 +237,7 @@ jobs:
- template: test_linux.yml
parameters:
AgentPool : 'Linux-GPU-CUDA10'
AgentPool : 'Onnxruntime-Linux-GPU'
TestGPU : 'true'
- job: Publish_NuGet_Package_And_Report

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-docker-release-pipeline.yml
View file

@ -16,7 +16,7 @@ jobs:
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-GPU-CUDA10
pool: Onnxruntime-Linux-GPU
steps:
- task: CmdLine@2
displayName: Build builder stage of docker file

6
tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml
View file

@ -297,7 +297,7 @@ jobs:
command: restore
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
@ -306,7 +306,7 @@ jobs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- template: win-esrp-dll.yml
@ -321,7 +321,7 @@ jobs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: BatchScript@1

6
tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
View file

@ -383,7 +383,7 @@ jobs:
command: restore
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
@ -392,7 +392,7 @@ jobs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- template: win-esrp-dll.yml
@ -407,7 +407,7 @@ jobs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj'
configuration: RelWithDebInfo
arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: BatchScript@1

2
tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
View file

@ -198,7 +198,7 @@ stages:
timeoutInMinutes: 120
workspace:
clean: all
pool: Linux-GPU-CUDA10
pool: Onnxruntime-Linux-GPU
strategy:
matrix:
Python36:

22
tools/ci_build/github/azure-pipelines/templates/win-ci.yml
View file

@ -22,37 +22,30 @@ parameters:
- name: EnvSetupScript
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
- name: buildArch
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
- name: msbuildPlatform
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
- name: packageName
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
- name: buildparameter
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
- name: runTests
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
type: boolean
default: true
- name: buildJava
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime'
type: boolean
default: true
- name: job_name_suffix
displayName: job name for nuget
@ -64,6 +57,11 @@ parameters:
type: string
default: 'Win-CPU-2021'
#'onnxruntime' or 'onnxruntime_gpu'
- name: java_artifact_id
type: string
default: 'onnxruntime'
jobs:
- job: Windows_Packaging_CPU_${{ parameters.job_name_suffix }}
workspace:
@ -181,7 +179,7 @@ jobs:
echo "Directories created"
copy .\java\build\libs\*.jar $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}
pushd $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}
set artifact_id=onnxruntime
set artifact_id=${{ parameters.java_artifact_id }}
jar xf onnxruntime-$(OnnxRuntimeVersion).jar META-INF\maven\com.microsoft.onnxruntime\%artifact_id%\pom.xml
move META-INF\maven\com.microsoft.onnxruntime\%artifact_id%\pom.xml onnxruntime-$(OnnxRuntimeVersion).pom
rd /s /q META-INF

12
tools/ci_build/github/windows/bundle_dlls.bat
View file

@ -12,12 +12,12 @@ FOR /R %%i IN (*.nupkg) do (
move onnxruntime-win-x86\lib\onnxruntime.dll runtimes\win-x86\native\onnxruntime.dll
move onnxruntime-win-x86\lib\onnxruntime.lib runtimes\win-x86\native\onnxruntime.lib
move onnxruntime-win-x86\lib\onnxruntime.pdb runtimes\win-x86\native\onnxruntime.pdb
move onnxruntime-win10-arm64\lib\onnxruntime.dll runtimes\win10-arm64\native\onnxruntime.dll
move onnxruntime-win10-arm64\lib\onnxruntime.lib runtimes\win10-arm64\native\onnxruntime.lib
move onnxruntime-win10-arm64\lib\onnxruntime.pdb runtimes\win10-arm64\native\onnxruntime.pdb
move onnxruntime-win10-arm\lib\onnxruntime.dll runtimes\win10-arm\native\onnxruntime.dll
move onnxruntime-win10-arm\lib\onnxruntime.lib runtimes\win10-arm\native\onnxruntime.lib
move onnxruntime-win10-arm\lib\onnxruntime.pdb runtimes\win10-arm\native\onnxruntime.pdb
move onnxruntime-win-arm64\lib\onnxruntime.dll runtimes\win10-arm64\native\onnxruntime.dll
move onnxruntime-win-arm64\lib\onnxruntime.lib runtimes\win10-arm64\native\onnxruntime.lib
move onnxruntime-win-arm64\lib\onnxruntime.pdb runtimes\win10-arm64\native\onnxruntime.pdb
move onnxruntime-win-arm\lib\onnxruntime.dll runtimes\win10-arm\native\onnxruntime.dll
move onnxruntime-win-arm\lib\onnxruntime.lib runtimes\win10-arm\native\onnxruntime.lib
move onnxruntime-win-arm\lib\onnxruntime.pdb runtimes\win10-arm\native\onnxruntime.pdb
move onnxruntime-linux-x64\lib\libonnxruntime.so.1* runtimes\linux-x64\native\libonnxruntime.so
move onnxruntime-osx-x64\lib\libonnxruntime.*.dylib runtimes\osx-x64\native\libonnxruntime.dylib
7z a %%~ni.nupkg runtimes

6
tools/ci_build/github/windows/jar_gpu_packaging.ps1
View file

@ -6,4 +6,10 @@ Write-Output "Run 7z"
7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.so
Remove-Item -Path libcustom_op_library.so
7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\onnxruntime-$Env:ONNXRUNTIMEVERSION.jar .
popd
pushd onnxruntime-java-win-x64
ren onnxruntime-$Env:ONNXRUNTIMEVERSION.jar onnxruntime_gpu-$Env:ONNXRUNTIMEVERSION.jar
ren onnxruntime-$Env:ONNXRUNTIMEVERSION-javadoc.jar onnxruntime_gpu-$Env:ONNXRUNTIMEVERSION-javadoc.jar
ren onnxruntime-$Env:ONNXRUNTIMEVERSION-sources.jar onnxruntime_gpu-$Env:ONNXRUNTIMEVERSION-sources.jar
ren onnxruntime-$Env:ONNXRUNTIMEVERSION.pom onnxruntime_gpu-$Env:ONNXRUNTIMEVERSION.pom
popd