mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-26 03:00:54 +00:00
Reduce downloads in Nuget-Java pipeline to reduce connection exception (#18635)
### Description 1. Add a new stage to download java tools from https://oss.sonatype.org and publish them to pipeline artifact 2. Remove downloads in other jobs, they get the java tools from pipeline artifact 3. consolidate final_java_testing stages. ### Motivation and Context Reduce downloads to reduce the connection error like below. ``` --2023-11-28 07:16:31-- https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar Resolving oss.sonatype.org (oss.sonatype.org)... 3.227.40.198, 3.229.50.23 Connecting to oss.sonatype.org (oss.sonatype.org)|3.227.40.198|:443... connected. HTTP request sent, awaiting response... 502 Bad Gateway 2023-11-28 07:16:32 ERROR 502: Bad Gateway. ```
This commit is contained in:
Yi Zhang
GitHub
parent
4025bd8ebd
commit
efee9abdb7
3 changed files with 178 additions and 166 deletions
|
|
@ -118,6 +118,30 @@ stages:
|
|||
- checkout: none
|
||||
- bash: echo $(MyVar)
|
||||
|
||||
- stage: Download_Java_Tools
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- job: Download_Java_Tools
|
||||
pool:
|
||||
vmImage: ubuntu-latest
|
||||
steps:
|
||||
- checkout: none
|
||||
- task: CmdLine@2
|
||||
displayName: Download Java Tools
|
||||
inputs:
|
||||
script: |
|
||||
mkdir -p java-tools
|
||||
pushd java-tools
|
||||
wget --tries=3 https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./
|
||||
wget --tries=3 https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -P ./
|
||||
popd
|
||||
workingDirectory: '$(Agent.TempDirectory)'
|
||||
- task: PublishPipelineArtifact@1
|
||||
displayName: 'Publish Pipeline Java Tools Artifact'
|
||||
inputs:
|
||||
targetPath: '$(Agent.TempDirectory)/java-tools'
|
||||
artifact: 'onnxruntime-java-tools'
|
||||
|
||||
- template: templates/c-api-cpu.yml
|
||||
parameters:
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
|
|
@ -309,6 +333,7 @@ stages:
|
|||
- Linux_C_API_Packaging_GPU_TensorRT_x64
|
||||
- Windows_Packaging_gpu
|
||||
- Windows_Packaging_tensorrt
|
||||
- Download_Java_Tools
|
||||
condition: succeeded()
|
||||
jobs:
|
||||
- job:
|
||||
|
|
@ -316,7 +341,6 @@ stages:
|
|||
clean: all
|
||||
pool: 'onnxruntime-Win-CPU-2022'
|
||||
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
submodules: false
|
||||
|
|
@ -398,12 +422,21 @@ stages:
|
|||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Final Jar'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'onnxruntime-java-gpu'
|
||||
targetPath: '$(Build.BinariesDirectory)\final-jar'
|
||||
- template: templates\flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Final Jar'
|
||||
ArtifactName: onnxruntime-java-gpu
|
||||
TargetPath: '$(Build.BinariesDirectory)\final-jar'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- template: templates\flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Jar Tools'
|
||||
ArtifactName: onnxruntime-java-tools
|
||||
TargetPath: '$(Build.BinariesDirectory)\final-jar'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
|
|
@ -412,8 +445,6 @@ stages:
|
|||
pushd test
|
||||
jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
|
||||
popd
|
||||
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
|
||||
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -OutFile protobuf-java-3.21.7.jar"
|
||||
java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.21.7.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
||||
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
|
||||
|
||||
|
|
|
|||
|
|
@ -213,6 +213,7 @@ stages:
|
|||
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
||||
- Windows_Packaging_CPU_arm_${{ parameters.BuildVariant }}
|
||||
- Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }}
|
||||
- Download_Java_Tools
|
||||
condition: succeeded()
|
||||
jobs:
|
||||
- job:
|
||||
|
|
@ -225,40 +226,45 @@ stages:
|
|||
submodules: false
|
||||
- template: set-version-number-variables-step.yml
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Pipeline Artifact - Win x64'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'drop-onnxruntime-java-win-x64'
|
||||
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Pipeline Artifact - Win x64'
|
||||
ArtifactName: 'drop-onnxruntime-java-win-x64'
|
||||
TargetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Pipeline Artifact - Linux x64'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'drop-onnxruntime-java-linux-x64'
|
||||
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64'
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Pipeline Artifact - Linux x64'
|
||||
ArtifactName: 'drop-onnxruntime-java-linux-x64'
|
||||
TargetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Pipeline Artifact - Linux AARCH64'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'drop-onnxruntime-java-linux-aarch64'
|
||||
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-aarch64'
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Pipeline Artifact - Linux AARCH64'
|
||||
ArtifactName: 'drop-onnxruntime-java-linux-aarch64'
|
||||
TargetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-aarch64'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Pipeline Artifact - MacOS x64'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'drop-onnxruntime-java-osx-x86_64'
|
||||
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64'
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Pipeline Artifact - MacOS x64'
|
||||
ArtifactName: 'drop-onnxruntime-java-osx-x86_64'
|
||||
TargetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Pipeline Artifact - MacOS ARM64'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'drop-onnxruntime-java-osx-arm64'
|
||||
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-arm64'
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Pipeline Artifact - MacOS ARM64'
|
||||
ArtifactName: 'drop-onnxruntime-java-osx-arm64'
|
||||
TargetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-arm64'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'PowerShell Script'
|
||||
|
|
@ -804,133 +810,24 @@ stages:
|
|||
- template: ../nodejs/templates/test_macos.yml
|
||||
parameters:
|
||||
StageSuffix : 'macOS_CPU_x64'
|
||||
- stage: Final_Jar_Testing_Windows
|
||||
dependsOn:
|
||||
Jar_Packaging
|
||||
jobs:
|
||||
- job:
|
||||
workspace:
|
||||
clean: all
|
||||
pool: 'onnxruntime-Win-CPU-2022'
|
||||
timeoutInMinutes: 60
|
||||
variables:
|
||||
- name: runCodesignValidationInjection
|
||||
value: false
|
||||
|
||||
steps:
|
||||
- template: set-version-number-variables-step.yml
|
||||
- template: final-jar-testing.yml
|
||||
parameters:
|
||||
OS: Windows
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
PoolName: 'onnxruntime-Win-CPU-2022'
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Final Jar'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'onnxruntime-java'
|
||||
targetPath: '$(Build.BinariesDirectory)\final-jar'
|
||||
- template: final-jar-testing.yml
|
||||
parameters:
|
||||
OS: Linux
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
PoolName: 'onnxruntime-Ubuntu2004-AMD-CPU'
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
mkdir test
|
||||
pushd test
|
||||
jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
|
||||
popd
|
||||
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
|
||||
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -OutFile protobuf-java-3.21.7.jar"
|
||||
java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.21.7.jar;onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
||||
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
- stage: Final_Jar_Testing_Linux
|
||||
dependsOn:
|
||||
Jar_Packaging
|
||||
jobs:
|
||||
- job:
|
||||
workspace:
|
||||
clean: all
|
||||
pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
|
||||
variables:
|
||||
- name: runCodesignValidationInjection
|
||||
value: false
|
||||
timeoutInMinutes: 60
|
||||
|
||||
steps:
|
||||
- template: set-version-number-variables-step.yml
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Final Jar'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'onnxruntime-java'
|
||||
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
echo "Java Version"
|
||||
java --version
|
||||
mkdir test
|
||||
pushd test
|
||||
jar xf $(Build.BinariesDirectory)/final-jar/testing.jar
|
||||
popd
|
||||
wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./
|
||||
wget https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -P ./
|
||||
LD_LIBRARY_PATH=./test:${LD_LIBRARY_PATH}
|
||||
java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.21.7.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
||||
workingDirectory: '$(Build.BinariesDirectory)/final-jar'
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
- stage: Final_Jar_Testing_MacOs
|
||||
dependsOn:
|
||||
Jar_Packaging
|
||||
jobs:
|
||||
- job:
|
||||
workspace:
|
||||
clean: all
|
||||
pool:
|
||||
vmImage: 'macOS-13'
|
||||
variables:
|
||||
- name: runCodesignValidationInjection
|
||||
value: false
|
||||
timeoutInMinutes: 60
|
||||
steps:
|
||||
- template: set-version-number-variables-step.yml
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download Final Jar'
|
||||
inputs:
|
||||
buildType: 'current'
|
||||
artifactName: 'onnxruntime-java'
|
||||
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
||||
|
||||
- template: use-xcode-version.yml
|
||||
|
||||
- task: CmdLine@2
|
||||
inputs:
|
||||
script: |
|
||||
echo "Java Version"
|
||||
java --version
|
||||
mkdir test
|
||||
pushd test
|
||||
jar xf $(Build.BinariesDirectory)/final-jar/testing.jar
|
||||
popd
|
||||
wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./
|
||||
wget https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -P ./
|
||||
DYLD_LIBRARY_PATH=./test:${DYLD_LIBRARY_PATH}
|
||||
java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.21.7.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
||||
workingDirectory: '$(Build.BinariesDirectory)/final-jar'
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
- template: final-jar-testing.yml
|
||||
parameters:
|
||||
OS: MacOS
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
PoolName: 'macOS-13'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
parameters:
|
||||
- name: OS
|
||||
displayName: Opserating System
|
||||
type: string
|
||||
|
||||
- name: SpecificArtifact
|
||||
displayName: Specific Artifact
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
- name: BuildId
|
||||
displayName: Build Id
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
- name: PoolName
|
||||
type: string
|
||||
|
||||
stages:
|
||||
- stage: Final_Jar_Testing_${{parameters.OS}}
|
||||
dependsOn:
|
||||
Jar_Packaging
|
||||
jobs:
|
||||
- job:
|
||||
workspace:
|
||||
clean: all
|
||||
${{ if eq(parameters.OS, 'MacOS') }}:
|
||||
pool:
|
||||
vmImage: ${{ parameters.PoolName }}
|
||||
${{ else }}:
|
||||
pool: ${{ parameters.PoolName }}
|
||||
variables:
|
||||
- name: runCodesignValidationInjection
|
||||
value: false
|
||||
timeoutInMinutes: 60
|
||||
|
||||
steps:
|
||||
- template: set-version-number-variables-step.yml
|
||||
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Final Jar'
|
||||
ArtifactName: onnxruntime-java
|
||||
TargetPath: '$(Build.BinariesDirectory)/final-jar'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- template: flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
StepName: 'Download Jar Tools'
|
||||
ArtifactName: onnxruntime-java-tools
|
||||
TargetPath: '$(Build.BinariesDirectory)/final-jar'
|
||||
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
||||
BuildId: ${{ parameters.BuildId }}
|
||||
|
||||
- task: Bash@3
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
echo "Java Version"
|
||||
java --version
|
||||
mkdir test
|
||||
pushd test
|
||||
jar xf '$(Build.BinariesDirectory)/final-jar/testing.jar'
|
||||
popd
|
||||
# if you want to run the tests in the power shell, you need to replace ':' to ';', that is, "-cp .;.\test;protobuf-java-3.21.7.jar;onnxruntime-$(OnnxRuntimeVersion).jar"
|
||||
java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.21.7.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
||||
workingDirectory: '$(Build.BinariesDirectory)/final-jar'
|
||||
env:
|
||||
${{ if eq(parameters.OS, 'MacOS') }}:
|
||||
DYLD_LIBRARY_PATH: '$(Build.BinariesDirectory)/final-jar/test:$(DYLD_LIBRARY_PATH)'
|
||||
${{ if eq(parameters.OS, 'Linux') }}:
|
||||
LD_LIBRARY_PATH: '$(Build.BinariesDirectory)/final-jar/test:$(LD_LIBRARY_PATH)'
|
||||
|
||||
- ${{ if eq(parameters['OS'], 'MacOS') }}:
|
||||
- template: use-xcode-version.yml
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
Loading…
Reference in a new issue