mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-25 22:26:24 +00:00
Combining android build and test step into one job (#21340)
### Description Combining android build and test step into one job ### Motivation and Context Reduce runtime by removing additional machine allocation, and artifact uploading and downloading. --------- Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
This commit is contained in:
Jian Chen
GitHub
parent
db9ee35963
commit
c03e6fff4c
2 changed files with 66 additions and 270 deletions
|
|
@ -48,12 +48,12 @@ parameters:
|
|||
stages:
|
||||
# Separate stage for building CPU vs NNAPI as we only want CodeQL to run on one of them so we don't get duplicate
|
||||
# issues for code that is built in both. We pick NNAPI as that includes the NNAPI EP code.
|
||||
- stage: BUILD_CPU_STAGE
|
||||
- stage: BUILD_AND_TEST_CPU
|
||||
dependsOn: []
|
||||
variables:
|
||||
Codeql.Enabled: false
|
||||
jobs:
|
||||
- job: Build_CPU_EP
|
||||
- job: BUILD_AND_TEST_CPU
|
||||
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
||||
workspace:
|
||||
clean: all
|
||||
|
|
@ -78,12 +78,14 @@ stages:
|
|||
- script: sudo apt-get update -y && sudo apt-get install -y coreutils ninja-build
|
||||
displayName: Install coreutils and ninja
|
||||
|
||||
- template: "templates/use-android-ndk.yml"
|
||||
|
||||
- template: templates/use-android-ndk.yml
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
create: true
|
||||
start: true
|
||||
- script: |
|
||||
env | grep ANDROID
|
||||
displayName: View Android ENVs
|
||||
|
||||
- script: |
|
||||
python3 tools/ci_build/build.py \
|
||||
--enable_lto \
|
||||
|
|
@ -96,42 +98,17 @@ stages:
|
|||
--skip_submodule_sync \
|
||||
--parallel \
|
||||
--cmake_generator=Ninja \
|
||||
--build_java \
|
||||
--skip_tests
|
||||
displayName: CPU EP, Build
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: Copy apks
|
||||
inputs:
|
||||
contents: 'build/**/*.apk'
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
overWrite: true
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: Copy test data
|
||||
inputs:
|
||||
contents: 'build/**/testdata/**'
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
overWrite: true
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: Copy test executables
|
||||
inputs:
|
||||
contents: |
|
||||
build/Debug/*
|
||||
build/Debug/java/androidtest/android/**
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
overWrite: true
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
inputs:
|
||||
pathToPublish: $(Build.ArtifactStagingDirectory)
|
||||
artifactName: CPUBuildOutput
|
||||
--build_java
|
||||
displayName: CPU EP, Build and Test
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
stop: true
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
|
||||
- stage: BUILD_NNAPI_STAGE
|
||||
- stage: BUILD_AND_TEST_NNAPI_EP
|
||||
dependsOn: []
|
||||
condition: notIn(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')
|
||||
variables:
|
||||
Codeql.ProjectConfigPath: .github/workflows
|
||||
Codeql.Enabled: true
|
||||
|
|
@ -140,14 +117,12 @@ stages:
|
|||
JobsTimeout: 120
|
||||
${{ else }}:
|
||||
JobsTimeout: 60
|
||||
|
||||
jobs:
|
||||
- job: Build_NNAPI_EP
|
||||
- job: BUILD_AND_TEST_NNAPI_EP
|
||||
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
||||
timeoutInMinutes: ${{ variables.JobsTimeout }}
|
||||
workspace:
|
||||
clean: all
|
||||
condition: notIn(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
displayName: Use Python $(pythonVersion)
|
||||
|
|
@ -163,8 +138,10 @@ stages:
|
|||
|
||||
- script: sudo apt-get update -y && sudo apt-get install -y coreutils ninja-build
|
||||
displayName: Install coreutils and ninja
|
||||
|
||||
- template: "templates/use-android-ndk.yml"
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
create: true
|
||||
start: true
|
||||
|
||||
- script: |
|
||||
env | grep ANDROID
|
||||
|
|
@ -172,194 +149,31 @@ stages:
|
|||
|
||||
- script: |
|
||||
python3 tools/ci_build/build.py \
|
||||
--enable_lto \
|
||||
--android \
|
||||
--build_dir build_nnapi \
|
||||
--android_sdk_path $ANDROID_HOME \
|
||||
--android_ndk_path $ANDROID_NDK_HOME \
|
||||
--android_abi=x86_64 \
|
||||
--android_api=29 \
|
||||
--skip_submodule_sync \
|
||||
--parallel \
|
||||
--use_nnapi \
|
||||
--cmake_generator=Ninja \
|
||||
--build_java \
|
||||
--skip_tests
|
||||
displayName: NNAPI EP, Build
|
||||
--enable_lto \
|
||||
--android \
|
||||
--build_dir build_nnapi \
|
||||
--android_sdk_path $ANDROID_HOME \
|
||||
--android_ndk_path $ANDROID_NDK_HOME \
|
||||
--android_abi=x86_64 \
|
||||
--android_api=29 \
|
||||
--skip_submodule_sync \
|
||||
--parallel \
|
||||
--use_nnapi \
|
||||
--build_shared_lib \
|
||||
--cmake_generator=Ninja \
|
||||
--build_java
|
||||
displayName: NNAPI EP, Build, Test on Android Emulator
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: Copy apks
|
||||
inputs:
|
||||
contents: 'build_nnapi/**/*.apk'
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
overWrite: true
|
||||
- script: /bin/bash tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh $(pwd)
|
||||
# Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator
|
||||
displayName: Build Minimal ORT with NNAPI and run tests
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: Copy test data
|
||||
inputs:
|
||||
contents: 'build_nnapi/**/testdata/**'
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
overWrite: true
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: Copy Test Executables
|
||||
inputs:
|
||||
contents: |
|
||||
build_nnapi/Debug/*
|
||||
build_nnapi/Debug/java/androidtest/android/**
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
overWrite: true
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
inputs:
|
||||
pathToPublish: $(Build.ArtifactStagingDirectory)
|
||||
artifactName: NNAPIBuildOutput
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
stop: true
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
|
||||
- stage: TEST_STAGE
|
||||
dependsOn: [BUILD_CPU_STAGE, BUILD_NNAPI_STAGE]
|
||||
jobs:
|
||||
- job: Test_CPU_EP
|
||||
pool:
|
||||
# We need macOS-12 to run the Android emulator for now.
|
||||
# https://github.com/actions/runner-images/issues/7671
|
||||
vmImage: 'macOS-12'
|
||||
workspace:
|
||||
clean: all
|
||||
condition: succeeded()
|
||||
steps:
|
||||
- script: |
|
||||
set -ex
|
||||
system_profiler SPSoftwareDataType SPHardwareDataType
|
||||
displayName: 'Mac Agent Info'
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
${{ if eq(parameters.specificArtifact, true) }}:
|
||||
source: 'specific'
|
||||
project: 'onnxruntime'
|
||||
pipeline: $(Build.DefinitionName)
|
||||
runVersion: 'specific'
|
||||
runId: ${{ parameters.runId }}
|
||||
${{ if ne(parameters.specificArtifact, true) }}:
|
||||
source: 'current'
|
||||
artifact: 'CPUBuildOutput'
|
||||
path: $(Build.SourcesDirectory)
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
displayName: Use Python $(pythonVersion)
|
||||
inputs:
|
||||
versionSpec: $(pythonVersion)
|
||||
|
||||
- task: JavaToolInstaller@0
|
||||
displayName: Use jdk 11
|
||||
inputs:
|
||||
versionSpec: '11'
|
||||
jdkArchitectureOption: 'x64'
|
||||
jdkSourceOption: 'PreInstalled'
|
||||
|
||||
- template: "templates/use-android-ndk.yml"
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
create: true
|
||||
start: true
|
||||
|
||||
- script: |
|
||||
python3 tools/ci_build/build.py \
|
||||
--enable_lto \
|
||||
--android \
|
||||
--build_dir build \
|
||||
--android_sdk_path $ANDROID_HOME \
|
||||
--android_ndk_path $ANDROID_NDK_HOME \
|
||||
--android_abi=x86_64 \
|
||||
--android_api=30 \
|
||||
--build_java \
|
||||
--test
|
||||
displayName: CPU EP, Test on Android Emulator
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
stop: true
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
|
||||
- job: Test_NNAPI_EP
|
||||
pool:
|
||||
# We need macOS-12 to run the Android emulator for now.
|
||||
# https://github.com/actions/runner-images/issues/7671
|
||||
vmImage: 'macOS-12'
|
||||
timeoutInMinutes: 90
|
||||
workspace:
|
||||
clean: all
|
||||
condition: and(succeeded(), notIn(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
|
||||
steps:
|
||||
- script: |
|
||||
set -ex
|
||||
system_profiler SPSoftwareDataType SPHardwareDataType
|
||||
displayName: 'Mac Agent Info'
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
${{ if eq(parameters.specificArtifact, true) }}:
|
||||
source: 'specific'
|
||||
project: 'onnxruntime'
|
||||
pipeline: $(Build.DefinitionName)
|
||||
runVersion: 'specific'
|
||||
runId: ${{ parameters.runId }}
|
||||
${{ if ne(parameters.specificArtifact, true) }}:
|
||||
source: 'current'
|
||||
artifact: 'NNAPIBuildOutput'
|
||||
path: $(Build.SourcesDirectory)
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
displayName: Use Python $(pythonVersion)
|
||||
inputs:
|
||||
versionSpec: $(pythonVersion)
|
||||
|
||||
- task: JavaToolInstaller@0
|
||||
displayName: Use jdk 11
|
||||
inputs:
|
||||
versionSpec: '11'
|
||||
jdkArchitectureOption: 'x64'
|
||||
jdkSourceOption: 'PreInstalled'
|
||||
|
||||
- template: "templates/use-android-ndk.yml"
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
create: true
|
||||
start: true
|
||||
|
||||
- script: |
|
||||
python3 tools/ci_build/build.py \
|
||||
--enable_lto \
|
||||
--android \
|
||||
--build_dir build_nnapi \
|
||||
--android_sdk_path $ANDROID_HOME \
|
||||
--android_ndk_path $ANDROID_NDK_HOME \
|
||||
--android_abi=x86_64 \
|
||||
--android_api=29 \
|
||||
--build_java \
|
||||
--use_nnapi \
|
||||
--test
|
||||
displayName: NNAPI EP, Test, CodeCoverage on Android Emulator
|
||||
|
||||
# used by Build Minimal ORT
|
||||
- script: brew install coreutils ninja
|
||||
displayName: Install coreutils and ninja
|
||||
|
||||
- script: /bin/bash tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh $(pwd)
|
||||
# Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator
|
||||
displayName: Build Minimal ORT with NNAPI and run tests
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
stop: true
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
|
||||
- stage: MASTER_BUILD_STAGE
|
||||
# The below jobs only run on master build.
|
||||
# because coverage report is hard to support in cross machines.
|
||||
|
|
@ -368,20 +182,12 @@ stages:
|
|||
condition: in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')
|
||||
jobs:
|
||||
- job: NNAPI_EP_MASTER
|
||||
pool:
|
||||
# We need macOS-12 to run the Android emulator for now.
|
||||
# https://github.com/actions/runner-images/issues/7671
|
||||
vmImage: 'macOS-12'
|
||||
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
||||
timeoutInMinutes: 180
|
||||
workspace:
|
||||
clean: all
|
||||
condition: in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')
|
||||
steps:
|
||||
- script: |
|
||||
set -ex
|
||||
system_profiler SPSoftwareDataType SPHardwareDataType
|
||||
displayName: 'Mac Agent Info'
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
displayName: Use Python $(pythonVersion)
|
||||
inputs:
|
||||
|
|
@ -394,11 +200,7 @@ stages:
|
|||
jdkArchitectureOption: 'x64'
|
||||
jdkSourceOption: 'PreInstalled'
|
||||
|
||||
- template: "templates/use-android-ndk.yml"
|
||||
|
||||
# used by Build Minimal ORT
|
||||
- script: brew install coreutils ninja
|
||||
displayName: Install coreutils and ninja
|
||||
- template: templates/use-android-ndk.yml
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
|
|
@ -429,50 +231,25 @@ stages:
|
|||
--build_dir build_nnapi \
|
||||
--android_sdk_path $ANDROID_HOME
|
||||
displayName: Retrieve runtime code coverage files from the emulator and analyze
|
||||
|
||||
- script: cat '$(Build.SourcesDirectory)/build_nnapi/Debug/coverage_rpt.txt'
|
||||
displayName: Print coverage report
|
||||
|
||||
- task: PublishPipelineArtifact@0
|
||||
displayName: 'Publish code coverage report'
|
||||
inputs:
|
||||
artifactName: "coverage_rpt.txt"
|
||||
targetPath: '$(Build.SourcesDirectory)/build_nnapi/Debug/coverage_rpt.txt'
|
||||
publishLocation: 'pipeline'
|
||||
|
||||
- script: /bin/bash tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh $(pwd)
|
||||
# Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator
|
||||
displayName: Build Minimal ORT with NNAPI and run tests
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
stop: true
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
|
||||
- job: Update_Dashboard
|
||||
workspace:
|
||||
clean: all
|
||||
variables:
|
||||
- name: skipComponentGovernanceDetection
|
||||
value: true
|
||||
pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
|
||||
condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
|
||||
dependsOn:
|
||||
- NNAPI_EP_MASTER
|
||||
steps:
|
||||
- task: DownloadPipelineArtifact@0
|
||||
displayName: 'Download code coverage report'
|
||||
inputs:
|
||||
artifactName: 'coverage_rpt.txt'
|
||||
targetPath: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: AzureCLI@2
|
||||
displayName: 'Post Android Code Coverage To DashBoard'
|
||||
inputs:
|
||||
azureSubscription: AIInfraBuild
|
||||
scriptType: bash
|
||||
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/github/linux/upload_code_coverage_data.sh
|
||||
arguments: '"$(Build.BinariesDirectory)/coverage_rpt.txt" "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" arm android nnapi'
|
||||
arguments: '"$(Build.SourcesDirectory)/build_nnapi/Debug/coverage_rpt.txt" "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" arm android nnapi'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- template: templates/use-android-emulator.yml
|
||||
parameters:
|
||||
stop: true
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
|
|
|
|||
|
|
@ -15,6 +15,25 @@ parameters:
|
|||
|
||||
steps:
|
||||
- ${{ if eq(parameters.create, true) }}:
|
||||
- script: |
|
||||
if [[ ":$PATH:" == *":${ANDROID_SDK_ROOT}/emulator:"* ]]; then
|
||||
echo "${ANDROID_SDK_ROOT}/emulator is in PATH"
|
||||
else
|
||||
${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager --install "emulator"
|
||||
echo "##vso[task.prependpath]${ANDROID_SDK_ROOT}/emulator"
|
||||
fi
|
||||
displayName: Check if emulator are installed and add to PATH
|
||||
|
||||
- script: |
|
||||
if [[ ":$PATH:" == *":${ANDROID_SDK_ROOT}/platform-tools:"* ]]; then
|
||||
echo "${ANDROID_SDK_ROOT}/platform-tools is in PATH"
|
||||
else
|
||||
${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager --install "platform-tools"
|
||||
echo "##vso[task.prependpath]${ANDROID_SDK_ROOT}/platform-tools"
|
||||
fi
|
||||
ls -R ${ANDROID_SDK_ROOT}/platform-tools
|
||||
displayName: Check if platform tools are installed and add to PATH
|
||||
|
||||
- script: |
|
||||
set -e -x
|
||||
python3 tools/python/run_android_emulator.py \
|
||||
|
|
|
|||
Loading…
Reference in a new issue