Refactor web-ci pipeline and delete eager mode CI pipeline (#15416)

### Description
1. Move it to a separated pool that use the same image as [the public
hosted
pool](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml).
Also, create a beta pool which contains the next version image of the
hosted pool, and add jobs in our post merge pipeline to test if the next
version image will break our CI. So, usually we will have at least one
week to prepare.

2. Change the cmake generator in use in our pipelines from "Ninja" to
"MingW Makefile", because the latest version of cmake doesn't work with
the latest version of Ninja. People who prefer Ninja could still use
ninja in their local build by passing "--cmake_generator ninja" to
[build.py](https://github.com/microsoft/onnxruntime/blob/main/tools/ci_build/build.py).

3. Delete eager mode CI pipeline. 


### Motivation and Context
I need to update the software we have in our CI build machines, and I
need to resolve this incompatibility issue. In more detail, the build
error I hit was:

em++: error:
CMakeFilesonnxruntime_mlas_test.dirC_a_work1sonnxruntimetestmlasunittesttest_activation.cpp.o:
No such file or directory
("CMakeFilesonnxruntime_mlas_test.dirC_a_work1sonnxruntimetestmlasunittesttest_activation.cpp.o"
was expected to be an input file, based on the commandline arguments
provided)

After this PR we will deprecate python 3.7 support. The eager mode CI
pipeline is the last one that still use python 3.7. Then we can rework
the PR #10953 made by [fs-eire](https://github.com/fs-eire) last year.

Fixed
[AB#14435](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/14435)
This commit is contained in:
Changming Sun 2023-04-10 10:41:04 -07:00 committed by GitHub
parent 9ef11f1c6a
commit c8524d2dab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 46 additions and 282 deletions

View file

@ -535,9 +535,16 @@ def parse_arguments():
)
parser.add_argument(
"--cmake_generator",
choices=["Visual Studio 15 2017", "Visual Studio 16 2019", "Visual Studio 17 2022", "Ninja"],
choices=[
"Visual Studio 16 2019",
"Visual Studio 17 2022",
"Ninja",
"MinGW Makefiles",
"NMake Makefiles",
"Xcode",
],
default="Visual Studio 16 2019" if is_windows() else None,
help="Specify the generator that CMake invokes. " "This is only supported on Windows",
help="Specify the generator that CMake invokes. ",
)
parser.add_argument(
"--enable_multi_device_test",
@ -642,15 +649,6 @@ def parse_arguments():
parser.add_argument("--ms_experimental", action="store_true", help="Build microsoft experimental operators.")
# eager mode
parser.add_argument("--build_eager_mode", action="store_true", help="Build ONNXRuntime micro-benchmarks.")
parser.add_argument(
"--eager_customop_module", default=None, help="Module containing custom op mappings for eager mode."
)
parser.add_argument(
"--eager_customop_header", default=None, help="Header containing custom op definitions for eager mode."
)
parser.add_argument(
"--enable_external_custom_op_schemas",
action="store_true",
@ -662,12 +660,6 @@ def parse_arguments():
"--external_graph_transformer_path", type=str, help="path to the external graph transformer dir."
)
parser.add_argument(
"--test_external_transformer_example",
action="store_true",
help="run the example external transformer test, mainly used in CI pipeline.",
)
parser.add_argument(
"--enable_cuda_profiling",
action="store_true",
@ -983,7 +975,6 @@ def generate_build_tree(
"-Donnxruntime_ENABLE_WEBASSEMBLY_THREADS=" + ("ON" if args.enable_wasm_threads else "OFF"),
"-Donnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO=" + ("ON" if args.enable_wasm_debug_info else "OFF"),
"-Donnxruntime_ENABLE_WEBASSEMBLY_PROFILING=" + ("ON" if args.enable_wasm_profiling else "OFF"),
"-Donnxruntime_ENABLE_EAGER_MODE=" + ("ON" if args.build_eager_mode else "OFF"),
"-Donnxruntime_ENABLE_LAZY_TENSOR=" + ("ON" if args.enable_lazy_tensor else "OFF"),
"-Donnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS="
+ ("ON" if args.enable_external_custom_op_schemas else "OFF"),
@ -1316,7 +1307,7 @@ def generate_build_tree(
else:
add_default_definition(cmake_extra_defines, "onnxruntime_PYBIND_EXPORT_OPSCHEMA", "OFF")
if args.build_eager_mode or args.enable_lazy_tensor:
if args.enable_lazy_tensor:
import torch
cmake_args += ["-Donnxruntime_PREBUILT_PYTORCH_PATH=%s" % os.path.dirname(torch.__file__)]
@ -1738,7 +1729,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
# Adding the torch lib path for loading DLLs for onnxruntime in eager mode
# This works for Python 3.7 and below, and doesn't work for Python 3.8+
# User will need to import torch before onnxruntime and it will work for all versions
if (args.build_eager_mode or args.enable_lazy_tensor) and is_windows():
if (args.enable_lazy_tensor) and is_windows():
import torch
dll_path_list.append(os.path.join(os.path.dirname(torch.__file__), "lib"))
@ -1804,28 +1795,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
if not args.disable_ml_ops and not args.use_tensorrt:
run_subprocess([sys.executable, "onnxruntime_test_python_mlops.py"], cwd=cwd, dll_path=dll_path)
if args.build_eager_mode:
# run eager mode test
args_list = [sys.executable, os.path.join(cwd, "eager_test")]
run_subprocess(args_list, cwd=cwd, dll_path=dll_path, python_path=cwd)
if args.test_external_transformer_example:
run_subprocess(
[
sys.executable,
os.path.join(
source_dir,
"orttraining",
"orttraining",
"test",
"external_transformer",
"test",
"external_transformers_test.py",
),
],
cwd=cwd,
dll_path=dll_path,
)
try:
import onnx # noqa: F401
@ -1940,7 +1909,6 @@ def build_python_wheel(
nightly_build=False,
default_training_package_device=False,
use_ninja=False,
build_eager_mode=False,
enable_training_apis=False,
enable_rocm_profiling=False,
):
@ -1962,8 +1930,6 @@ def build_python_wheel(
args.append("--enable_training")
if enable_training_apis:
args.append("--enable_training_apis")
if build_eager_mode:
args.append("--disable_auditwheel_repair")
if enable_rocm_profiling:
args.append("--enable_rocm_profiling")
@ -2473,11 +2439,9 @@ def main():
raise BuildError("The value to --path_to_protoc_exe is invalid.")
if not args.skip_submodule_sync:
update_submodules(source_dir)
if is_windows():
if is_windows() and not args.build_wasm:
cpu_arch = platform.architecture()[0]
if args.build_wasm:
cmake_extra_args = ["-G", "Ninja"]
elif args.cmake_generator == "Ninja":
if args.cmake_generator == "Ninja":
if cpu_arch == "32bit" or args.arm or args.arm64 or args.arm64ec:
raise BuildError(
"To cross-compile with Ninja, load the toolset "
@ -2563,71 +2527,6 @@ def main():
if args.use_rocm and args.rocm_version is None:
args.rocm_version = ""
if args.build_eager_mode:
eager_root_dir = os.path.join(source_dir, "orttraining", "orttraining", "eager")
if args.eager_customop_module and not args.eager_customop_header:
raise Exception("eager_customop_header must be provided when eager_customop_module is")
elif args.eager_customop_header and not args.eager_customop_module:
raise Exception("eager_customop_module must be provided when eager_customop_header is")
def gen_ops(gen_cpp_name: str, header_file: str, ops_module: str, custom_ops: bool):
gen_cpp_scratch_name = gen_cpp_name + ".working"
print(
f"Generating ORT ATen overrides (output_file: {gen_cpp_name}, header_file: {header_file},"
f"ops_module: {ops_module}), custom_ops: {custom_ops}"
)
cmd = [
sys.executable,
os.path.join(os.path.join(eager_root_dir, "opgen", "opgen.py")),
"--output_file",
gen_cpp_scratch_name,
"--ops_module",
ops_module,
"--header_file",
header_file,
]
if custom_ops:
cmd += ["--custom_ops"]
subprocess.check_call(cmd)
import filecmp
if not os.path.isfile(gen_cpp_name) or not filecmp.cmp(
gen_cpp_name, gen_cpp_scratch_name, shallow=False
):
os.rename(gen_cpp_scratch_name, gen_cpp_name)
else:
os.remove(gen_cpp_scratch_name)
def gen_ort_ops():
# generate native aten ops
import torch
regdecs_path = os.path.join(os.path.dirname(torch.__file__), "include/ATen/RegistrationDeclarations.h")
ops_module = os.path.join(eager_root_dir, "opgen/opgen/atenops.py")
gen_ops(os.path.join(eager_root_dir, "ort_aten.g.cpp"), regdecs_path, ops_module, False)
# generate custom ops
if not args.eager_customop_header:
args.eager_customop_header = os.path.realpath(
os.path.join(eager_root_dir, "opgen", "CustomOpDeclarations.h")
)
if not args.eager_customop_module:
args.eager_customop_module = os.path.join(eager_root_dir, "opgen/opgen/custom_ops.py")
gen_ops(
os.path.join(eager_root_dir, "ort_customops.g.cpp"),
args.eager_customop_header,
args.eager_customop_module,
True,
)
gen_ort_ops()
if args.enable_external_custom_op_schemas and not is_linux():
raise BuildError("Registering external custom op schemas is only supported on Linux.")
@ -2718,7 +2617,6 @@ def main():
nightly_build=nightly_build,
default_training_package_device=default_training_package_device,
use_ninja=(args.cmake_generator == "Ninja"),
build_eager_mode=args.build_eager_mode,
enable_training_apis=args.enable_training_apis,
enable_rocm_profiling=args.enable_rocm_profiling,
)

View file

@ -20,7 +20,7 @@ stages:
- stage: BUILD_STAGE
jobs:
- job: Build_CPU_EP
pool: onnxruntime-Linux-CPU-2019
pool: onnxruntime-Linux-CPU-For-Android-CI
workspace:
clean: all
timeoutInMinutes: 30
@ -105,7 +105,7 @@ stages:
- template: templates/clean-agent-build-directory-step.yml
- job: Build_NNAPI_EP
pool: onnxruntime-Linux-CPU-2019
pool: onnxruntime-Linux-CPU-For-Android-CI
timeoutInMinutes: 60
workspace:
clean: all
@ -452,7 +452,7 @@ stages:
variables:
- name: skipComponentGovernanceDetection
value: true
pool: 'onnxruntime-Linux-CPU-2019'
pool: 'onnxruntime-Linux-CPU-For-Android-CI'
condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
dependsOn:
- NNAPI_EP_MASTER

View file

@ -7,71 +7,6 @@ resources:
ref: aead4d751c2101e23336aa73f2380df83e7a13f3
jobs:
- job: BuildAndTestEagerMode
timeoutInMinutes: 120
workspace:
clean: all
pool: onnxruntime-Linux-CPU-2019
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_eager_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecpubuildeagermode
- task: CmdLine@2
displayName: 'build'
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecpubuildeagermode \
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel \
--build_eager_mode --enable_training --build_wheel --skip_test
workingDirectory: $(Build.SourcesDirectory)
- task: CmdLine@2
displayName: 'install ortmodule extension and test'
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimecpubuildeagermode \
bash -c "export PYTHONPATH=/build/Release && /opt/python/cp38-cp38/bin/python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install &&\
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel \
--build_eager_mode --enable_training --build_wheel --test"
workingDirectory: $(Build.SourcesDirectory)
- template: templates/clean-agent-build-directory-step.yml
# This pipeline builds the latest PyTorch commit from source
# and use it in ORT tests. See Dockerfile.manylinux2014_lort_cpu
# for the installation steps. Idally, we should only use one pipeline

View file

@ -41,8 +41,9 @@ stages:
parameters:
NpmPackagingMode: ${{ variables.NpmPackagingMode }}
IsReleasePipeline: true
PoolName: 'Win-CPU-2021'
PoolName: 'Azure-Pipelines-EO-Windows2022-aiinfra'
PackageName: 'onnxruntime-web'
ExtraBuildArgs: ''
- template: templates/react-native-ci.yml
parameters:
@ -59,7 +60,7 @@ stages:
- Build_web_Debug
jobs:
- job: Download_Node_Package
pool: 'Win-CPU-2021'
pool: 'Azure-Pipelines-EO-Windows2022-aiinfra'
variables:
runCodesignValidationInjection: false
timeoutInMinutes: 10

View file

@ -15,6 +15,15 @@ variables:
value: 'Linux-CPU'
stages:
- ${{ if or(startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/aiinfra/'),startsWith(variables['System.CollectionUri'], 'https://aiinfra.visualstudio.com/')) }}:
- template: templates/web-ci.yml
parameters:
NpmPackagingMode: 'dev'
IsReleasePipeline: true
PoolName: 'aiinfra-Win-CPU-2022-web-beta'
BuildStaticLib: true
ExtraBuildArgs: ''
# This stage is to test if the combined build works on
# o Windows ARM64
# o Windows ARM64EC

View file

@ -31,7 +31,7 @@ stages:
timeoutInMinutes: 60
workspace:
clean: all
pool: onnxruntime-Linux-CPU-2019
pool: onnxruntime-Linux-CPU-For-Android-CI
steps:
- checkout: self
clean: true

View file

@ -20,6 +20,10 @@ parameters:
type: boolean
default: false
- name: ExtraBuildArgs
type: string
stages:
- stage: Extract_commit
jobs:
@ -50,7 +54,7 @@ stages:
parameters:
CommitOverride: true
BuildConfig: 'Debug'
ExtraBuildArgs: '--use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON $(ExtraBuildArgs)'
ExtraBuildArgs: '--use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}'
PoolName: ${{ parameters.PoolName }}
- stage: Build_web_Debug
@ -71,7 +75,7 @@ stages:
parameters:
CommitOverride: true
BuildConfig: 'Release'
ExtraBuildArgs: '--skip_tests --enable_wasm_api_exception_catching --disable_rtti --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON $(ExtraBuildArgs)'
ExtraBuildArgs: '--skip_tests --enable_wasm_api_exception_catching --disable_rtti --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}'
PoolName: ${{ parameters.PoolName }}
- ${{ if eq(parameters.BuildStaticLib, 'true') }}:
@ -82,7 +86,7 @@ stages:
parameters:
CommitOverride: true
BuildConfig: 'Release'
ExtraBuildArgs: '$(ExtraBuildArgs)'
ExtraBuildArgs: '${{ parameters.ExtraBuildArgs }}'
PoolName: ${{ parameters.PoolName }}
SkipPublish: true
TimeoutInMinutes: 270

View file

@ -29,7 +29,7 @@ jobs:
variables:
EnvSetupScript: setup_env.bat
buildArch: x64
CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wasm --use_xnnpack --emsdk_version releases-29ad1037cd6b99e5d8a1bd75bc188c1e9a6fda8d-64bit ${{ parameters.ExtraBuildArgs }}'
CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --cmake_generator "MinGW Makefiles" --build_wasm --use_xnnpack --emsdk_version releases-29ad1037cd6b99e5d8a1bd75bc188c1e9a6fda8d-64bit ${{ parameters.ExtraBuildArgs }}'
runCodesignValidationInjection: false
timeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
workspace:
@ -62,16 +62,14 @@ jobs:
- task: NodeTool@0
inputs:
versionSpec: '16.x'
- task: BatchScript@1
displayName: 'setup env'
- template: download-deps.yml
- task: PythonScript@0
displayName: 'Update deps.txt'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q setuptools wheel numpy ninja
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
arguments: --new_dir $(Build.BinariesDirectory)/deps
workingDirectory: $(Build.BinariesDirectory)
- task: PythonScript@0
displayName: 'Build and test (node)'

View file

@ -28,5 +28,6 @@ stages:
parameters:
NpmPackagingMode: ${{ variables.NpmPackagingMode }}
IsReleasePipeline: false
PoolName: 'onnxruntime-Win-CPU-2019'
PoolName: 'onnxruntime-Win-CPU-2022-web'
BuildStaticLib: true
ExtraBuildArgs: $(ExtraBuildArgs)

View file

@ -1,82 +0,0 @@
parameters:
- name: RunOnnxRuntimeTests
displayName: Run Tests?
type: boolean
default: true
jobs:
- job: 'build'
pool: 'Win-CPU-2019'
strategy:
matrix:
x64_release:
BuildConfig: 'Release'
EnvSetupScript: setup_env.bat
buildArch: x64
additionalBuildFlags: --use_dnnl --build_java --build_nodejs
msbuildPlatform: x64
isX86: false
variables:
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
MsbuildArguments: '-maxcpucount'
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
setVcvars: true
ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
timeoutInMinutes: 120
workspace:
clean: all
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7'
addToPath: true
architecture: $(buildArch)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
set ORT_DOXY_SRC=$(Build.SourcesDirectory)
set ORT_DOXY_OUT=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)
mkdir %ORT_DOXY_SRC%
mkdir %ORT_DOXY_OUT%
"C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg
workingDirectory: '$(Build.SourcesDirectory)'
displayName: 'API Documentation Check and generate'
- script: |
python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\windows\eager\requirements.txt
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: BatchScript@1
displayName: 'setup env 2'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: PythonScript@0
displayName: 'Build Eager mode'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --build_eager_mode --enable_training --build_wheel --parallel'
workingDirectory: '$(Build.BinariesDirectory)'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**/*.results.xml'
searchFolder: '$(Build.BinariesDirectory)/$(BuildConfig)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'