mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
f68a256140
* Added Onnxruntime_GCOV_COVERAGE flag for Android. * Set CMAKE_SYSTEM_NAME explicityly for Android. * Added GCOV_PREFIX option to collect code coverage data. Added a new python script to generate code coverage info. Modified build pipeline to geneate Android code coverage info * Added build command line option --android_coverage * Added a comment describing the GCOV environment variables * Fixed PEP8 issues. * Added --android_coverage option to the build command. * Increased Android emulator memory from 3K to 8K. * Increased Android partition-size from 2GB to 4GB to overcome no-space-left-on-device error * Removed source_dir from command line args. * Use cwd absolute path to run tests. * Added commands to output the contents of /data/local/tmp on the emulator. * Added run_adb_shell function. * Format changes. * Removed keywd argument cwd. * Removed Android in the --build_dir path. * Removed commands added for debugging. * Removed exxtra new-lines. * Fix MacOs build pipeline failures by uninstalling openssl before running build script. * Revert "Fix MacOs build pipeline failures by uninstalling openssl before running build script." This reverts commit 90d0568fe533e9456c20d061a2d435c8fea48266. * Change dir to the build directory where the tar file is copied. * Changed the option from --android_coverage to --code_coverage * Moved steps to generate Android code coverage to run_nnap_code_coverage.sh * Require --android option if --code_coverage is specified. * No code coverage needed for onnx_test_runner. * Expect that the emulator is running when the script is executed. * Fixed the title in the buildpipeline step. * Fixed the formatting issue. * Added a command line argument, ORT_ROOT, to run_nnapi_code_coverage.sh script Co-authored-by: Satya Jandhyala <satyajandhyala@Satyas-Mac-mini.local>
2113 lines
90 KiB
Python
2113 lines
90 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
# Licensed under the MIT License.
|
|
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import hashlib
|
|
import platform
|
|
from logger import get_logger
|
|
from amd_hipify import amd_hipify
|
|
from distutils.version import LooseVersion
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
|
|
|
|
sys.path.append(os.path.join(REPO_DIR, "tools", "python"))
|
|
|
|
|
|
from util import run # noqa: E402
|
|
|
|
|
|
log = get_logger("build")
|
|
|
|
|
|
class BaseError(Exception):
|
|
"""Base class for errors originating from build.py."""
|
|
pass
|
|
|
|
|
|
class BuildError(BaseError):
|
|
"""Error from running build steps."""
|
|
|
|
def __init__(self, *messages):
|
|
super().__init__("\n".join(messages))
|
|
|
|
|
|
class UsageError(BaseError):
|
|
"""Usage related error."""
|
|
|
|
def __init__(self, message):
|
|
super().__init__(message)
|
|
|
|
|
|
def _check_python_version():
|
|
# According to the BUILD.md, python 3.5+ is required:
|
|
# Python 2 is definitely not supported and it should be safer to consider
|
|
# it won't run with python 4:
|
|
if sys.version_info[0] != 3:
|
|
raise BuildError(
|
|
"Bad python major version: expecting python 3, found version "
|
|
"'{}'".format(sys.version))
|
|
if sys.version_info[1] < 5:
|
|
raise BuildError(
|
|
"Bad python minor version: expecting python 3.5+, found version "
|
|
"'{}'".format(sys.version))
|
|
|
|
|
|
_check_python_version()
|
|
|
|
|
|
def _openvino_verify_device_type(device_read):
|
|
choices = ["CPU_FP32", "GPU_FP32", "GPU_FP16", "VAD-M_FP16", "MYRIAD_FP16", "VAD-F_FP32"]
|
|
status_hetero = True
|
|
res = False
|
|
if (device_read in choices):
|
|
res = True
|
|
elif (device_read.startswith("HETERO:") or device_read.startswith("MULTI:")):
|
|
res = True
|
|
comma_separated_devices = device_read.split(":")
|
|
comma_separated_devices = comma_separated_devices[1].split(',')
|
|
if (len(comma_separated_devices) < 2):
|
|
print("Atleast two devices required in Hetero Mode")
|
|
status_hetero = False
|
|
dev_options = ["CPU", "GPU", "MYRIAD", "FPGA", "HDDL"]
|
|
for dev in comma_separated_devices:
|
|
if (dev not in dev_options):
|
|
status_hetero = False
|
|
break
|
|
|
|
def invalid_hetero_build():
|
|
print("\n" + "If trying to build Hetero or Multi, specifiy the supported devices along with it." + + "\n")
|
|
print("specify the keyword HETERO or MULTI followed by the devices ")
|
|
print("in the order of priority you want to build" + "\n")
|
|
print("The different hardware devices that can be added in HETERO or MULTI")
|
|
print("are ['CPU','GPU','MYRIAD','FPGA','HDDL']" + "\n")
|
|
print("An example of how to specify the hetero build type. Ex: HETERO:GPU,CPU" + "\n")
|
|
print("An example of how to specify the MULTI build type. Ex: MULTI:MYRIAD,CPU" + "\n")
|
|
sys.exit("Wrong Build Type selected")
|
|
|
|
if (res is False):
|
|
print("\n" + "You have selcted wrong configuration for the build.")
|
|
print("pick the build type for specific Hardware Device from following options: ", choices)
|
|
print("\n")
|
|
if not (device_read.startswith("HETERO:") or device_read.startswith("MULTI:")):
|
|
invalid_hetero_build()
|
|
sys.exit("Wrong Build Type selected")
|
|
|
|
if (status_hetero is False):
|
|
invalid_hetero_build()
|
|
|
|
return device_read
|
|
|
|
|
|
def parse_arguments():
|
|
parser = argparse.ArgumentParser(
|
|
description="ONNXRuntime CI build driver.",
|
|
usage=""" # noqa
|
|
Default behavior is --update --build --test for native architecture builds.
|
|
Default behavior is --update --build for cross-compiled builds.
|
|
|
|
The Update phase will update git submodules, and run cmake to generate makefiles.
|
|
The Build phase will build all projects.
|
|
The Test phase will run all unit tests, and optionally the ONNX tests.
|
|
|
|
Use the individual flags to only run the specified stages.
|
|
""")
|
|
# Main arguments
|
|
parser.add_argument(
|
|
"--build_dir", required=True, help="Path to the build directory.")
|
|
parser.add_argument(
|
|
"--config", nargs="+", default=["Debug"],
|
|
choices=["Debug", "MinSizeRel", "Release", "RelWithDebInfo"],
|
|
help="Configuration(s) to build.")
|
|
parser.add_argument(
|
|
"--update", action='store_true', help="Update makefiles.")
|
|
parser.add_argument("--build", action='store_true', help="Build.")
|
|
parser.add_argument(
|
|
"--clean", action='store_true',
|
|
help="Run 'cmake --build --target clean' for the selected config/s.")
|
|
parser.add_argument(
|
|
"--parallel", nargs='?', const='0', default='1', type=int,
|
|
help="Use parallel build. The optional value specifies the maximum number of parallel jobs. "
|
|
"If the optional value is 0 or unspecified, it is interpreted as the number of CPUs.")
|
|
parser.add_argument("--test", action='store_true', help="Run unit tests.")
|
|
parser.add_argument(
|
|
"--skip_tests", action='store_true', help="Skip all tests.")
|
|
|
|
# Training options
|
|
parser.add_argument(
|
|
"--enable_nvtx_profile", action='store_true', help="Enable NVTX profile in ORT.")
|
|
parser.add_argument(
|
|
"--enable_training", action='store_true', help="Enable training in ORT.")
|
|
parser.add_argument(
|
|
"--enable_training_python_frontend_e2e_tests", action="store_true",
|
|
help="Enable the pytorch frontend training tests.")
|
|
parser.add_argument(
|
|
"--enable_training_pipeline_e2e_tests", action="store_true",
|
|
help="Enable the pipeline c++ e2e tests.")
|
|
parser.add_argument(
|
|
"--use_horovod", action='store_true', help="Enable Horovod.")
|
|
parser.add_argument(
|
|
"--disable_nccl", action='store_true', help="Disable Nccl.")
|
|
parser.add_argument(
|
|
"--mpi_home", help="Path to MPI installation dir")
|
|
parser.add_argument(
|
|
"--nccl_home", help="Path to NCCL installation dir")
|
|
|
|
# enable ONNX tests
|
|
parser.add_argument(
|
|
"--enable_onnx_tests", action='store_true',
|
|
help="""When running the Test phase, run onnx_test_running against
|
|
available test data directories.""")
|
|
parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe.")
|
|
parser.add_argument(
|
|
"--fuzz_testing", action='store_true', help="Enable Fuzz testing of the onnxruntime.")
|
|
parser.add_argument(
|
|
"--enable_symbolic_shape_infer_tests", action='store_true',
|
|
help="""When running the Test phase, run symbolic shape inference against
|
|
available test data directories.""")
|
|
|
|
# generate documentaiton
|
|
parser.add_argument(
|
|
"--gen_doc", action='store_true',
|
|
help="Generate documentation on contrib ops")
|
|
|
|
# CUDA related
|
|
parser.add_argument("--use_cuda", action='store_true', help="Enable CUDA.")
|
|
parser.add_argument(
|
|
"--cuda_version", help="The version of CUDA toolkit to use. "
|
|
"Auto-detect if not specified. e.g. 9.0")
|
|
parser.add_argument(
|
|
"--cuda_home", help="Path to CUDA home."
|
|
"Read from CUDA_HOME environment variable if --use_cuda is true and "
|
|
"--cuda_home is not specified.")
|
|
parser.add_argument(
|
|
"--cudnn_home", help="Path to CUDNN home. "
|
|
"Read from CUDNN_HOME environment variable if --use_cuda is true and "
|
|
"--cudnn_home is not specified.")
|
|
|
|
# Python bindings
|
|
parser.add_argument(
|
|
"--enable_pybind", action='store_true', help="Enable Python Bindings.")
|
|
parser.add_argument(
|
|
"--build_wheel", action='store_true', help="Build Python Wheel.")
|
|
parser.add_argument(
|
|
"--wheel_name_suffix", help="Suffix to append to created wheel names. "
|
|
"This value is currently only used for nightly builds.")
|
|
parser.add_argument(
|
|
"--numpy_version", help="Installs a specific version of numpy "
|
|
"before building the python binding.")
|
|
parser.add_argument(
|
|
"--skip-keras-test", action='store_true',
|
|
help="Skip tests with Keras if keras is installed")
|
|
|
|
# C-Sharp bindings
|
|
parser.add_argument(
|
|
"--build_csharp", action='store_true',
|
|
help="Build C#.Net DLL and NuGet package. This should be only used in CI pipelines. "
|
|
"For building C# bindings and packaging them into nuget package use --build_nuget arg.")
|
|
|
|
parser.add_argument(
|
|
"--build_nuget", action='store_true',
|
|
help="Build C#.Net DLL and NuGet package on the local machine. "
|
|
"Currently only Windows and Linux platforms are supported.")
|
|
|
|
# Java bindings
|
|
parser.add_argument(
|
|
"--build_java", action='store_true', help="Build Java bindings.")
|
|
|
|
# Node.js binding
|
|
parser.add_argument(
|
|
"--build_nodejs", action='store_true',
|
|
help="Build Node.js binding and NPM package.")
|
|
|
|
# Build a shared lib
|
|
parser.add_argument(
|
|
"--build_shared_lib", action='store_true',
|
|
help="Build a shared library for the ONNXRuntime.")
|
|
|
|
# Build options
|
|
parser.add_argument(
|
|
"--cmake_extra_defines", nargs="+",
|
|
help="Extra definitions to pass to CMake during build system "
|
|
"generation. These are just CMake -D options without the leading -D.")
|
|
parser.add_argument(
|
|
"--target",
|
|
help="Build a specific target, e.g. winml_dll")
|
|
parser.add_argument(
|
|
"--x86", action='store_true',
|
|
help="Create x86 makefiles. Requires --update and no existing cache "
|
|
"CMake setup. Delete CMakeCache.txt if needed")
|
|
parser.add_argument(
|
|
"--arm", action='store_true',
|
|
help="Create ARM makefiles. Requires --update and no existing cache "
|
|
"CMake setup. Delete CMakeCache.txt if needed")
|
|
parser.add_argument(
|
|
"--arm64", action='store_true',
|
|
help="Create ARM64 makefiles. Requires --update and no existing cache "
|
|
"CMake setup. Delete CMakeCache.txt if needed")
|
|
parser.add_argument(
|
|
"--msvc_toolset", help="MSVC toolset to use. e.g. 14.11")
|
|
parser.add_argument("--android", action='store_true', help='Build for Android')
|
|
parser.add_argument(
|
|
"--android_abi", default="arm64-v8a",
|
|
choices=["armeabi-v7a", "arm64-v8a", "x86", "x86_64"],
|
|
help="Specify the target Android Application Binary Interface (ABI)")
|
|
parser.add_argument("--android_api", type=int, default=27, help='Android API Level, e.g. 21')
|
|
parser.add_argument("--android_sdk_path", type=str, help='Path to the Android SDK')
|
|
parser.add_argument("--android_ndk_path", default="", help="Path to the Android NDK")
|
|
parser.add_argument("--android_cpp_shared", action="store_true",
|
|
help="Build with shared libc++ instead of the default static libc++.")
|
|
parser.add_argument("--test_binary_size", action="store_true",
|
|
help="If enabled, build will fail when the built binary size is larger than the threshold. "
|
|
"This only applies to Android Minimal build for now.")
|
|
|
|
parser.add_argument("--ios", action='store_true', help="build for ios")
|
|
parser.add_argument(
|
|
"--ios_sysroot", default="",
|
|
help="Specify the location name of the macOS platform SDK to be used")
|
|
parser.add_argument(
|
|
"--ios_toolchain_dir", default="",
|
|
help="Path to ios toolchain binaries")
|
|
parser.add_argument(
|
|
"--ios_toolchain_file", default="",
|
|
help="Path to ios toolchain file, "
|
|
"or cmake/onnxruntime_ios.toolchain.cmake will be used")
|
|
parser.add_argument(
|
|
"--xcode_code_signing_team_id", default="",
|
|
help="The development team ID used for code signing in Xcode")
|
|
parser.add_argument(
|
|
"--use_xcode", action='store_true',
|
|
help="Use Xcode as cmake generator, this is only supported on MacOS.")
|
|
parser.add_argument(
|
|
"--osx_arch",
|
|
default="arm64" if platform.machine() == "arm64" else "x86_64",
|
|
choices=["arm64", "x86_64"],
|
|
help="Specify the Target specific architectures for macOS and iOS, This is only supported on MacOS")
|
|
parser.add_argument(
|
|
"--apple_deploy_target", type=str,
|
|
help="Specify the minimum version of the target platform "
|
|
"(e.g. macOS or iOS)"
|
|
"This is only supported on MacOS")
|
|
|
|
# Arguments needed by CI
|
|
parser.add_argument(
|
|
"--cmake_path", default="cmake", help="Path to the CMake program.")
|
|
parser.add_argument(
|
|
"--ctest_path", default="ctest", help="Path to the CTest program.")
|
|
parser.add_argument(
|
|
"--skip_submodule_sync", action='store_true', help="Don't do a "
|
|
"'git submodule update'. Makes the Update phase faster.")
|
|
parser.add_argument(
|
|
"--use_vstest", action='store_true',
|
|
help="Use use_vstest for running unitests.")
|
|
parser.add_argument(
|
|
"--use_jemalloc", action='store_true', help="Use jemalloc.")
|
|
parser.add_argument(
|
|
"--use_mimalloc", default=['none'],
|
|
choices=['none', 'stl', 'arena', 'all'], help="Use mimalloc.")
|
|
parser.add_argument(
|
|
"--use_openblas", action='store_true', help="Build with OpenBLAS.")
|
|
parser.add_argument(
|
|
"--use_dnnl", action='store_true', help="Build with DNNL.")
|
|
parser.add_argument(
|
|
"--dnnl_gpu_runtime", action='store', default='', type=str.lower,
|
|
help="e.g. --dnnl_gpu_runtime ocl")
|
|
parser.add_argument(
|
|
"--dnnl_opencl_root", action='store', default='',
|
|
help="Path to OpenCL SDK. "
|
|
"e.g. --dnnl_opencl_root \"C:/Program Files (x86)/IntelSWTools/sw_dev_tools/OpenCL/sdk\"")
|
|
parser.add_argument(
|
|
"--use_mklml", action='store_true', help="Build with MKLML.")
|
|
parser.add_argument(
|
|
"--use_featurizers", action='store_true',
|
|
help="Build with ML Featurizer support.")
|
|
parser.add_argument(
|
|
"--use_openvino", nargs="?", const="CPU_FP32",
|
|
type=_openvino_verify_device_type,
|
|
help="Build with OpenVINO for specific hardware.")
|
|
parser.add_argument(
|
|
"--use_nnapi", action='store_true', help="Build with NNAPI support.")
|
|
parser.add_argument(
|
|
"--nnapi_min_api", type=int,
|
|
help="Minimum Android API level to enable NNAPI, should be no less than 27")
|
|
parser.add_argument(
|
|
"--use_rknpu", action='store_true', help="Build with RKNPU.")
|
|
parser.add_argument(
|
|
"--use_preinstalled_eigen", action='store_true',
|
|
help="Use pre-installed Eigen.")
|
|
parser.add_argument("--eigen_path", help="Path to pre-installed Eigen.")
|
|
parser.add_argument(
|
|
"--use_openmp", action='store_true', help="Build with OpenMP")
|
|
parser.add_argument(
|
|
"--enable_msinternal", action="store_true",
|
|
help="Enable for Microsoft internal builds only.")
|
|
parser.add_argument("--llvm_path", help="Path to llvm dir")
|
|
parser.add_argument(
|
|
"--use_vitisai", action='store_true', help="Build with Vitis-AI")
|
|
parser.add_argument(
|
|
"--use_nuphar", action='store_true', help="Build with nuphar")
|
|
parser.add_argument(
|
|
"--use_tensorrt", action='store_true', help="Build with TensorRT")
|
|
parser.add_argument(
|
|
"--tensorrt_home", help="Path to TensorRT installation dir")
|
|
parser.add_argument(
|
|
"--use_migraphx", action='store_true', help="Build with MIGraphX")
|
|
parser.add_argument(
|
|
"--migraphx_home", help="Path to MIGraphX installation dir")
|
|
parser.add_argument(
|
|
"--use_full_protobuf", action='store_true',
|
|
help="Use the full protobuf library")
|
|
|
|
parser.add_argument(
|
|
"--skip_onnx_tests", action='store_true', help="Explicitly disable "
|
|
"all onnx related tests. Note: Use --skip_tests to skip all tests.")
|
|
parser.add_argument(
|
|
"--skip_winml_tests", action='store_true',
|
|
help="Explicitly disable all WinML related tests")
|
|
parser.add_argument(
|
|
"--skip_nodejs_tests", action='store_true',
|
|
help="Explicitly disable all Node.js binding tests")
|
|
parser.add_argument(
|
|
"--enable_msvc_static_runtime", action='store_true',
|
|
help="Enable static linking of MSVC runtimes.")
|
|
parser.add_argument(
|
|
"--enable_language_interop_ops", action='store_true',
|
|
help="Enable operator implemented in language other than cpp")
|
|
parser.add_argument(
|
|
"--cmake_generator",
|
|
choices=['Visual Studio 15 2017', 'Visual Studio 16 2019', 'Ninja'],
|
|
default='Visual Studio 15 2017' if is_windows() else None,
|
|
help="Specify the generator that CMake invokes. "
|
|
"This is only supported on Windows")
|
|
parser.add_argument(
|
|
"--enable_multi_device_test", action='store_true',
|
|
help="Test with multi-device. Mostly used for multi-device GPU")
|
|
parser.add_argument(
|
|
"--use_dml", action='store_true', help="Build with DirectML.")
|
|
parser.add_argument(
|
|
"--use_winml", action='store_true', help="Build with WinML.")
|
|
parser.add_argument(
|
|
"--winml_root_namespace_override", type=str,
|
|
help="Specify the namespace that WinML builds into.")
|
|
parser.add_argument(
|
|
"--use_telemetry", action='store_true',
|
|
help="Only official builds can set this flag to enable telemetry.")
|
|
parser.add_argument(
|
|
"--enable_wcos", action='store_true',
|
|
help="Build for Windows Core OS.")
|
|
parser.add_argument(
|
|
"--enable_windows_store", action='store_true',
|
|
help="Build for Windows Store")
|
|
parser.add_argument(
|
|
"--enable_lto", action='store_true',
|
|
help="Enable Link Time Optimization")
|
|
parser.add_argument(
|
|
"--use_acl", nargs="?", const="ACL_1905",
|
|
choices=["ACL_1902", "ACL_1905", "ACL_1908", "ACL_2002"],
|
|
help="Build with ACL for ARM architectures.")
|
|
parser.add_argument(
|
|
"--acl_home", help="Path to ACL home dir")
|
|
parser.add_argument(
|
|
"--acl_libs", help="Path to ACL libraries")
|
|
parser.add_argument(
|
|
"--use_armnn", action='store_true',
|
|
help="Enable ArmNN Execution Provider.")
|
|
parser.add_argument(
|
|
"--armnn_relu", action='store_true',
|
|
help="Use the Relu operator implementation from the ArmNN EP.")
|
|
parser.add_argument(
|
|
"--armnn_bn", action='store_true',
|
|
help="Use the Batch Normalization operator implementation from the ArmNN EP.")
|
|
parser.add_argument(
|
|
"--armnn_home", help="Path to ArmNN home dir")
|
|
parser.add_argument(
|
|
"--armnn_libs", help="Path to ArmNN libraries")
|
|
parser.add_argument(
|
|
"--build_micro_benchmarks", action='store_true',
|
|
help="Build ONNXRuntime micro-benchmarks.")
|
|
|
|
# options to reduce binary size
|
|
parser.add_argument("--minimal_build", action='store',
|
|
const='on', default='off', nargs='?', type=str.lower,
|
|
help="Create a build that only supports ORT format models. "
|
|
"See /docs/ONNX_Runtime_Format_Model_Usage.md for more information. "
|
|
"RTTI is automatically disabled in a minimal build. "
|
|
"To enable execution providers that compile kernels at runtime (e.g. NNAPI) pass 'extended' "
|
|
"as a parameter. e.g. '--minimal_build extended'.")
|
|
parser.add_argument("--include_ops_by_model", type=str, help="include ops from model(s) under designated path.")
|
|
parser.add_argument("--include_ops_by_config", type=str,
|
|
help="include ops from config file. "
|
|
"See /docs/Reduced_Operator_Kernel_build.md for more information.")
|
|
|
|
parser.add_argument("--disable_contrib_ops", action='store_true',
|
|
help="Disable contrib ops (reduces binary size)")
|
|
parser.add_argument("--disable_ml_ops", action='store_true',
|
|
help="Disable traditional ML ops (reduces binary size)")
|
|
parser.add_argument("--disable_rtti", action='store_true', help="Disable RTTI (reduces binary size)")
|
|
parser.add_argument("--disable_exceptions", action='store_true',
|
|
help="Disable exceptions to reduce binary size. Requires --minimal_build.")
|
|
parser.add_argument("--disable_ort_format_load", action='store_true',
|
|
help='Disable support for loading ORT format models in a non-minimal build.')
|
|
|
|
parser.add_argument("--use_rocm", action='store_true', help="Build with ROCm")
|
|
parser.add_argument("--rocm_home", help="Path to ROCm installation dir")
|
|
|
|
# Code coverage
|
|
parser.add_argument("--code_coverage", action='store_true',
|
|
help="Generate code coverage when targetting Android (only).")
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def resolve_executable_path(command_or_path):
|
|
"""Returns the absolute path of an executable."""
|
|
executable_path = shutil.which(command_or_path)
|
|
if executable_path is None:
|
|
raise BuildError("Failed to resolve executable path for "
|
|
"'{}'.".format(command_or_path))
|
|
return os.path.realpath(executable_path)
|
|
|
|
|
|
def is_windows():
|
|
return sys.platform.startswith("win")
|
|
|
|
|
|
def is_macOS():
|
|
return sys.platform.startswith("darwin")
|
|
|
|
|
|
def is_linux():
|
|
return sys.platform.startswith("linux")
|
|
|
|
|
|
def get_linux_distro():
|
|
try:
|
|
with open('/etc/os-release', 'r') as f:
|
|
dist_info = dict(
|
|
line.strip().split('=', 1) for line in f.readlines())
|
|
return dist_info.get('NAME', '').strip('"'), dist_info.get(
|
|
'VERSION', '').strip('"')
|
|
except (IOError, ValueError):
|
|
return '', ''
|
|
|
|
|
|
def is_ubuntu_1604():
|
|
dist, ver = get_linux_distro()
|
|
return dist == 'Ubuntu' and ver.startswith('16.04')
|
|
|
|
|
|
def get_config_build_dir(build_dir, config):
|
|
# build directory per configuration
|
|
return os.path.join(build_dir, config)
|
|
|
|
|
|
def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None,
|
|
shell=False, env={}):
|
|
if isinstance(args, str):
|
|
raise ValueError("args should be a sequence of strings, not a string")
|
|
|
|
my_env = os.environ.copy()
|
|
if dll_path:
|
|
if is_windows():
|
|
my_env["PATH"] = dll_path + os.pathsep + my_env["PATH"]
|
|
else:
|
|
if "LD_LIBRARY_PATH" in my_env:
|
|
my_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path
|
|
else:
|
|
my_env["LD_LIBRARY_PATH"] = dll_path
|
|
|
|
my_env.update(env)
|
|
|
|
return run(*args, cwd=cwd, capture_stdout=capture_stdout, shell=shell, env=my_env)
|
|
|
|
|
|
def update_submodules(source_dir):
|
|
run_subprocess(["git", "submodule", "sync", "--recursive"], cwd=source_dir)
|
|
run_subprocess(["git", "submodule", "update", "--init", "--recursive"],
|
|
cwd=source_dir)
|
|
|
|
|
|
def is_docker():
|
|
path = '/proc/self/cgroup'
|
|
return (
|
|
os.path.exists('/.dockerenv') or
|
|
os.path.isfile(path) and any('docker' in line for line in open(path))
|
|
)
|
|
|
|
|
|
def is_sudo():
|
|
return 'SUDO_UID' in os.environ.keys()
|
|
|
|
|
|
def install_apt_package(package):
|
|
have = package in str(run_subprocess(
|
|
["apt", "list", "--installed", package], capture_stdout=True).stdout)
|
|
if not have:
|
|
if is_sudo():
|
|
run_subprocess(['apt-get', 'install', '-y', package])
|
|
else:
|
|
raise BuildError(package + " APT package missing. Please re-run "
|
|
"this script using sudo to install.")
|
|
|
|
|
|
def install_ubuntu_deps(args):
|
|
"""Check if the necessary Ubuntu dependencies are installed.
|
|
Not required on docker. Provide help output if missing."""
|
|
|
|
# check we need the packages first
|
|
if not (args.enable_pybind or args.use_openblas):
|
|
return
|
|
|
|
# not needed on docker as packages are pre-installed
|
|
if not is_docker():
|
|
try:
|
|
if args.enable_pybind:
|
|
install_apt_package("python3")
|
|
|
|
if args.use_openblas:
|
|
install_apt_package("libopenblas-dev")
|
|
|
|
except Exception as e:
|
|
raise BuildError("Error setting up required APT packages. "
|
|
"{}".format(str(e)))
|
|
|
|
|
|
def install_python_deps(numpy_version=""):
|
|
dep_packages = ['setuptools', 'wheel', 'pytest']
|
|
dep_packages.append('numpy=={}'.format(numpy_version) if numpy_version
|
|
else 'numpy>=1.16.6')
|
|
dep_packages.append('sympy>=1.1')
|
|
dep_packages.append('packaging')
|
|
dep_packages.append('cerberus')
|
|
run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host',
|
|
'files.pythonhosted.org'] + dep_packages)
|
|
|
|
|
|
# We need to install Torch to test certain functionalities of the ORT Python package
|
|
def install_torch():
|
|
# Command works for both Windows
|
|
run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host',
|
|
'files.pythonhosted.org', 'torch===1.5.1+cu101', 'torchvision===0.6.1+cu101',
|
|
'-f', 'https://download.pytorch.org/whl/torch_stable.html'])
|
|
|
|
|
|
def check_md5(filename, expected_md5):
|
|
if not os.path.exists(filename):
|
|
return False
|
|
hash_md5 = hashlib.md5()
|
|
BLOCKSIZE = 1024*64
|
|
with open(filename, "rb") as f:
|
|
buf = f.read(BLOCKSIZE)
|
|
while len(buf) > 0:
|
|
hash_md5.update(buf)
|
|
buf = f.read(BLOCKSIZE)
|
|
hex = hash_md5.hexdigest()
|
|
if hex != expected_md5:
|
|
log.info('md5 mismatch, expect %s, got %s' % (expected_md5, hex))
|
|
os.remove(filename)
|
|
return False
|
|
return True
|
|
|
|
|
|
def setup_test_data(build_dir, configs):
|
|
# create a shortcut for test models if there is a 'models'
|
|
# folder in build_dir
|
|
if is_windows():
|
|
src_model_dir = os.path.join(build_dir, 'models')
|
|
if os.path.exists('C:\\local\\models') and not os.path.exists(
|
|
src_model_dir):
|
|
log.debug("creating shortcut %s -> %s" % (
|
|
'C:\\local\\models', src_model_dir))
|
|
run_subprocess(['mklink', '/D', '/J', src_model_dir,
|
|
'C:\\local\\models'], shell=True)
|
|
for config in configs:
|
|
config_build_dir = get_config_build_dir(build_dir, config)
|
|
os.makedirs(config_build_dir, exist_ok=True)
|
|
dest_model_dir = os.path.join(config_build_dir, 'models')
|
|
if os.path.exists('C:\\local\\models') and not os.path.exists(
|
|
dest_model_dir):
|
|
log.debug("creating shortcut %s -> %s" % (
|
|
'C:\\local\\models', dest_model_dir))
|
|
run_subprocess(['mklink', '/D', '/J', dest_model_dir,
|
|
'C:\\local\\models'], shell=True)
|
|
elif os.path.exists(src_model_dir) and not os.path.exists(
|
|
dest_model_dir):
|
|
log.debug("creating shortcut %s -> %s" % (
|
|
src_model_dir, dest_model_dir))
|
|
run_subprocess(['mklink', '/D', '/J', dest_model_dir,
|
|
src_model_dir], shell=True)
|
|
|
|
|
|
def use_dev_mode(args):
|
|
if args.use_acl:
|
|
return 'OFF'
|
|
if args.use_armnn:
|
|
return 'OFF'
|
|
if args.ios and is_macOS():
|
|
return 'OFF'
|
|
return 'ON'
|
|
|
|
|
|
def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, rocm_home,
|
|
mpi_home, nccl_home, tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs,
|
|
path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
|
|
log.info("Generating CMake build tree")
|
|
cmake_dir = os.path.join(source_dir, "cmake")
|
|
# TODO: fix jemalloc build so it does not conflict with onnxruntime
|
|
# shared lib builds. (e.g. onnxuntime_pybind)
|
|
# for now, disable jemalloc if pybind is also enabled.
|
|
cmake_args = [
|
|
cmake_path, cmake_dir,
|
|
"-Donnxruntime_RUN_ONNX_TESTS=" + (
|
|
"ON" if args.enable_onnx_tests else "OFF"),
|
|
"-Donnxruntime_BUILD_WINML_TESTS=" + (
|
|
"OFF" if args.skip_winml_tests else "ON"),
|
|
"-Donnxruntime_GENERATE_TEST_REPORTS=ON",
|
|
"-Donnxruntime_DEV_MODE=" + use_dev_mode(args),
|
|
"-DPYTHON_EXECUTABLE=" + sys.executable,
|
|
"-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"),
|
|
"-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""),
|
|
"-Donnxruntime_USE_FEATURIZERS=" + (
|
|
"ON" if args.use_featurizers else "OFF"),
|
|
"-Donnxruntime_CUDA_HOME=" + (cuda_home if args.use_cuda else ""),
|
|
"-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"),
|
|
"-Donnxruntime_USE_MIMALLOC_STL_ALLOCATOR=" + (
|
|
"ON" if args.use_mimalloc == "stl" or
|
|
args.use_mimalloc == "all" else "OFF"),
|
|
"-Donnxruntime_USE_MIMALLOC_ARENA_ALLOCATOR=" + (
|
|
"ON" if args.use_mimalloc == "arena" or
|
|
args.use_mimalloc == "all" else "OFF"),
|
|
"-Donnxruntime_ENABLE_PYTHON=" + (
|
|
"ON" if args.enable_pybind else "OFF"),
|
|
"-Donnxruntime_BUILD_CSHARP=" + ("ON" if args.build_csharp else "OFF"),
|
|
"-Donnxruntime_BUILD_JAVA=" + ("ON" if args.build_java else "OFF"),
|
|
"-Donnxruntime_BUILD_NODEJS=" + ("ON" if args.build_nodejs else "OFF"),
|
|
"-Donnxruntime_BUILD_SHARED_LIB=" + (
|
|
"ON" if args.build_shared_lib else "OFF"),
|
|
"-Donnxruntime_USE_EIGEN_FOR_BLAS=" + (
|
|
"OFF" if args.use_openblas else "ON"),
|
|
"-Donnxruntime_USE_OPENBLAS=" + ("ON" if args.use_openblas else "OFF"),
|
|
"-Donnxruntime_USE_DNNL=" + ("ON" if args.use_dnnl else "OFF"),
|
|
"-Donnxruntime_DNNL_GPU_RUNTIME=" + (args.dnnl_gpu_runtime if args.use_dnnl else ""),
|
|
"-Donnxruntime_DNNL_OPENCL_ROOT=" + (args.dnnl_opencl_root if args.use_dnnl else ""),
|
|
"-Donnxruntime_USE_MKLML=" + ("ON" if args.use_mklml else "OFF"),
|
|
"-Donnxruntime_USE_NNAPI_BUILTIN=" + ("ON" if args.use_nnapi else "OFF"),
|
|
"-Donnxruntime_USE_RKNPU=" + ("ON" if args.use_rknpu else "OFF"),
|
|
"-Donnxruntime_USE_OPENMP=" + (
|
|
"ON" if args.use_openmp and not (
|
|
args.use_nnapi or (args.use_mklml and (is_macOS() or is_windows())) or
|
|
args.android or (args.ios and is_macOS())
|
|
or args.use_rknpu)
|
|
else "OFF"),
|
|
"-Donnxruntime_USE_TVM=" + ("ON" if args.use_nuphar else "OFF"),
|
|
"-Donnxruntime_USE_LLVM=" + ("ON" if args.use_nuphar else "OFF"),
|
|
"-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + (
|
|
"ON" if args.enable_msinternal else "OFF"),
|
|
"-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"),
|
|
"-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"),
|
|
"-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"),
|
|
"-Donnxruntime_TENSORRT_HOME=" + (
|
|
tensorrt_home if args.use_tensorrt else ""),
|
|
# set vars for migraphx
|
|
"-Donnxruntime_USE_MIGRAPHX=" + ("ON" if args.use_migraphx else "OFF"),
|
|
"-Donnxruntime_MIGRAPHX_HOME=" + (migraphx_home if args.use_migraphx else ""),
|
|
# By default - we currently support only cross compiling for
|
|
# ARM/ARM64 (no native compilation supported through this
|
|
# script).
|
|
"-Donnxruntime_CROSS_COMPILING=" + (
|
|
"ON" if args.arm64 or args.arm else "OFF"),
|
|
"-Donnxruntime_DISABLE_CONTRIB_OPS=" + ("ON" if args.disable_contrib_ops else "OFF"),
|
|
"-Donnxruntime_DISABLE_ML_OPS=" + ("ON" if args.disable_ml_ops else "OFF"),
|
|
"-Donnxruntime_DISABLE_RTTI=" + ("ON" if args.disable_rtti else "OFF"),
|
|
"-Donnxruntime_DISABLE_EXCEPTIONS=" + ("ON" if args.disable_exceptions else "OFF"),
|
|
"-Donnxruntime_DISABLE_ORT_FORMAT_LOAD=" + ("ON" if args.disable_ort_format_load else "OFF"),
|
|
"-Donnxruntime_MINIMAL_BUILD=" + ("ON" if args.minimal_build != 'off' else "OFF"),
|
|
"-Donnxruntime_EXTENDED_MINIMAL_BUILD=" + ("ON" if args.minimal_build == 'extended' else "OFF"),
|
|
"-Donnxruntime_REDUCED_OPS_BUILD=" + (
|
|
"ON" if args.include_ops_by_config or args.include_ops_by_model else "OFF"),
|
|
"-Donnxruntime_MSVC_STATIC_RUNTIME=" + (
|
|
"ON" if args.enable_msvc_static_runtime else "OFF"),
|
|
# enable pyop if it is nightly build
|
|
"-Donnxruntime_ENABLE_LANGUAGE_INTEROP_OPS=" + (
|
|
"ON" if args.enable_language_interop_ops else "OFF"),
|
|
"-Donnxruntime_USE_DML=" + ("ON" if args.use_dml else "OFF"),
|
|
"-Donnxruntime_USE_WINML=" + ("ON" if args.use_winml else "OFF"),
|
|
"-Donnxruntime_USE_TELEMETRY=" + (
|
|
"ON" if args.use_telemetry else "OFF"),
|
|
"-Donnxruntime_ENABLE_LTO=" + ("ON" if args.enable_lto else "OFF"),
|
|
"-Donnxruntime_USE_ACL=" + ("ON" if args.use_acl else "OFF"),
|
|
"-Donnxruntime_USE_ACL_1902=" + (
|
|
"ON" if args.use_acl == "ACL_1902" else "OFF"),
|
|
"-Donnxruntime_USE_ACL_1905=" + (
|
|
"ON" if args.use_acl == "ACL_1905" else "OFF"),
|
|
"-Donnxruntime_USE_ACL_1908=" + (
|
|
"ON" if args.use_acl == "ACL_1908" else "OFF"),
|
|
"-Donnxruntime_USE_ACL_2002=" + (
|
|
"ON" if args.use_acl == "ACL_2002" else "OFF"),
|
|
"-Donnxruntime_USE_ARMNN=" + (
|
|
"ON" if args.use_armnn else "OFF"),
|
|
"-Donnxruntime_ARMNN_RELU_USE_CPU=" + (
|
|
"OFF" if args.armnn_relu else "ON"),
|
|
"-Donnxruntime_ARMNN_BN_USE_CPU=" + (
|
|
"OFF" if args.armnn_bn else "ON"),
|
|
# Training related flags
|
|
"-Donnxruntime_ENABLE_NVTX_PROFILE=" + (
|
|
"ON" if args.enable_nvtx_profile else "OFF"),
|
|
"-Donnxruntime_ENABLE_TRAINING=" + (
|
|
"ON" if args.enable_training else "OFF"),
|
|
"-Donnxruntime_USE_HOROVOD=" + (
|
|
"ON" if args.use_horovod else "OFF"),
|
|
"-Donnxruntime_USE_NCCL=" + (
|
|
"OFF" if args.disable_nccl else "ON"),
|
|
"-Donnxruntime_BUILD_BENCHMARKS=" + (
|
|
"ON" if args.build_micro_benchmarks else "OFF"),
|
|
"-Donnxruntime_USE_ROCM=" + ("ON" if args.use_rocm else "OFF"),
|
|
"-Donnxruntime_ROCM_HOME=" + (rocm_home if args.use_rocm else ""),
|
|
"-DOnnxruntime_GCOV_COVERAGE=" + ("ON" if args.code_coverage else "OFF"),
|
|
]
|
|
|
|
if acl_home and os.path.exists(acl_home):
|
|
cmake_args += ["-Donnxruntime_ACL_HOME=" + acl_home]
|
|
|
|
if acl_libs and os.path.exists(acl_libs):
|
|
cmake_args += ["-Donnxruntime_ACL_LIBS=" + acl_libs]
|
|
|
|
if armnn_home and os.path.exists(armnn_home):
|
|
cmake_args += ["-Donnxruntime_ARMNN_HOME=" + armnn_home]
|
|
|
|
if armnn_libs and os.path.exists(armnn_libs):
|
|
cmake_args += ["-Donnxruntime_ARMNN_LIBS=" + armnn_libs]
|
|
|
|
if mpi_home and os.path.exists(mpi_home):
|
|
cmake_args += ["-Donnxruntime_MPI_HOME=" + mpi_home]
|
|
|
|
if nccl_home and os.path.exists(nccl_home):
|
|
cmake_args += ["-Donnxruntime_NCCL_HOME=" + nccl_home]
|
|
|
|
if args.winml_root_namespace_override:
|
|
cmake_args += ["-Donnxruntime_WINML_NAMESPACE_OVERRIDE=" +
|
|
args.winml_root_namespace_override]
|
|
if args.use_openvino:
|
|
cmake_args += ["-Donnxruntime_USE_OPENVINO=ON",
|
|
"-Donnxruntime_USE_OPENVINO_MYRIAD=" + (
|
|
"ON" if args.use_openvino == "MYRIAD_FP16" else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_GPU_FP32=" + (
|
|
"ON" if args.use_openvino == "GPU_FP32" else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_GPU_FP16=" + (
|
|
"ON" if args.use_openvino == "GPU_FP16" else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_CPU_FP32=" + (
|
|
"ON" if args.use_openvino == "CPU_FP32" else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_VAD_M=" + (
|
|
"ON" if args.use_openvino == "VAD-M_FP16" else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_VAD_F=" + (
|
|
"ON" if args.use_openvino == "VAD-F_FP32" else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_HETERO=" + (
|
|
"ON" if args.use_openvino.startswith("HETERO") else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_DEVICE=" + (args.use_openvino),
|
|
"-Donnxruntime_USE_OPENVINO_MULTI=" + (
|
|
"ON" if args.use_openvino.startswith("MULTI") else "OFF"),
|
|
"-Donnxruntime_USE_OPENVINO_BINARY=" + (
|
|
"ON" if args.use_openvino else "OFF")]
|
|
# temp turn on only for linux gpu build
|
|
if not is_windows():
|
|
if args.use_cuda:
|
|
cmake_args += [
|
|
"-Donnxruntime_USE_FULL_PROTOBUF=ON"]
|
|
|
|
# TensorRT and OpenVINO providers currently only supports
|
|
# full_protobuf option.
|
|
if (args.use_full_protobuf or args.use_tensorrt or
|
|
args.use_openvino or args.use_vitisai or args.gen_doc):
|
|
cmake_args += [
|
|
"-Donnxruntime_USE_FULL_PROTOBUF=ON",
|
|
"-DProtobuf_USE_STATIC_LIBS=ON"
|
|
]
|
|
|
|
if args.use_nuphar and args.llvm_path is not None:
|
|
cmake_args += ["-DLLVM_DIR=%s" % args.llvm_path]
|
|
|
|
if args.use_cuda and not is_windows():
|
|
nvml_stub_path = cuda_home + "/lib64/stubs"
|
|
cmake_args += ["-DCUDA_CUDA_LIBRARY=" + nvml_stub_path]
|
|
|
|
if args.use_preinstalled_eigen:
|
|
cmake_args += ["-Donnxruntime_USE_PREINSTALLED_EIGEN=ON",
|
|
"-Deigen_SOURCE_PATH=" + args.eigen_path]
|
|
|
|
if args.nnapi_min_api:
|
|
cmake_args += ["-Donnxruntime_NNAPI_MIN_API=" + str(args.nnapi_min_api)]
|
|
|
|
if args.android:
|
|
cmake_args += [
|
|
"-DCMAKE_TOOLCHAIN_FILE=" + args.android_ndk_path + "/build/cmake/android.toolchain.cmake",
|
|
"-DANDROID_PLATFORM=android-" + str(args.android_api),
|
|
"-DANDROID_ABI=" + str(args.android_abi)
|
|
]
|
|
|
|
if args.android_cpp_shared:
|
|
cmake_args += ["-DANDROID_STL=c++_shared"]
|
|
|
|
if is_macOS() and not args.android:
|
|
cmake_args += ["-DCMAKE_OSX_ARCHITECTURES=" + args.osx_arch]
|
|
# since cmake 3.19, it uses the xcode latest buildsystem, which is not supported by this project.
|
|
cmake_verstr = subprocess.check_output(['cmake', '--version']).decode('utf-8').split()[2]
|
|
if args.use_xcode and LooseVersion(cmake_verstr) >= LooseVersion('3.19.0'):
|
|
cmake_args += ["-T", "buildsystem=1"]
|
|
|
|
if args.ios:
|
|
if is_macOS():
|
|
needed_args = [
|
|
args.use_xcode,
|
|
args.ios_sysroot,
|
|
args.apple_deploy_target,
|
|
]
|
|
arg_names = [
|
|
"--use_xcode " +
|
|
"<need use xcode to cross build iOS on MacOS>",
|
|
"--ios_sysroot " +
|
|
"<the location or name of the macOS platform SDK>",
|
|
"--apple_deploy_target " +
|
|
"<the minimum version of the target platform>",
|
|
]
|
|
if not all(needed_args):
|
|
raise BuildError(
|
|
"iOS build on MacOS canceled due to missing arguments: " +
|
|
', '.join(
|
|
val for val, cond in zip(arg_names, needed_args)
|
|
if not cond))
|
|
cmake_args += [
|
|
"-DCMAKE_SYSTEM_NAME=iOS",
|
|
"-Donnxruntime_BUILD_SHARED_LIB=ON",
|
|
"-DCMAKE_OSX_SYSROOT=" + args.ios_sysroot,
|
|
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + args.apple_deploy_target,
|
|
# we do not need protoc binary for ios cross build
|
|
"-Dprotobuf_BUILD_PROTOC_BINARIES=OFF",
|
|
"-DCMAKE_TOOLCHAIN_FILE=" + (
|
|
args.ios_toolchain_file if args.ios_toolchain_file
|
|
else "../cmake/onnxruntime_ios.toolchain.cmake")
|
|
]
|
|
# Code sign the binaries, if the code signing development team id is provided
|
|
if args.xcode_code_signing_team_id:
|
|
cmake_args += ["-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=" + args.xcode_code_signing_team_id]
|
|
else:
|
|
# TODO: the cross compiling on Linux is not officially supported by Apple
|
|
# and is already broken with the latest codebase, so it should be removed.
|
|
# We are cross compiling on Linux
|
|
needed_args = [
|
|
args.ios_sysroot,
|
|
args.arm64 or args.arm,
|
|
args.ios_toolchain_dir
|
|
]
|
|
arg_names = [
|
|
"--ios_sysroot <path to sysroot>",
|
|
"--arm or --arm64",
|
|
"--ios_toolchain_dir <path to toolchain>"
|
|
]
|
|
if not all(needed_args):
|
|
raise BuildError(
|
|
"iOS build canceled due to missing arguments: " +
|
|
', '.join(
|
|
val for val, cond in zip(arg_names, needed_args)
|
|
if not cond))
|
|
compilers = sorted(
|
|
glob.glob(args.ios_toolchain_dir + "/bin/*-clang*"))
|
|
os.environ["PATH"] = os.path.join(
|
|
args.ios_toolchain_dir, "bin") + os.pathsep + os.environ.get(
|
|
"PATH", "")
|
|
os.environ["LD_LIBRARY_PATH"] = os.path.join(
|
|
args.ios_toolchain_dir, "/lib") + os.pathsep + os.environ.get(
|
|
"LD_LIBRARY_PATH", "")
|
|
if len(compilers) != 2:
|
|
raise BuildError(
|
|
"error identifying compilers in ios_toolchain_dir")
|
|
cmake_args += [
|
|
"-DCMAKE_OSX_ARCHITECTURES=" +
|
|
("arm64" if args.arm64 else "arm"),
|
|
"-DCMAKE_SYSTEM_NAME=iOSCross",
|
|
"-Donnxruntime_BUILD_UNIT_TESTS=OFF",
|
|
"-DCMAKE_OSX_SYSROOT=" + args.ios_sysroot,
|
|
"-DCMAKE_C_COMPILER=" + compilers[0],
|
|
"-DCMAKE_CXX_COMPILER=" + compilers[1]
|
|
]
|
|
|
|
if path_to_protoc_exe:
|
|
cmake_args += [
|
|
"-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % path_to_protoc_exe]
|
|
|
|
if args.fuzz_testing:
|
|
if not (args.build_shared_lib and
|
|
is_windows() and
|
|
args.cmake_generator == 'Visual Studio 16 2019' and
|
|
args.use_full_protobuf):
|
|
raise BuildError(
|
|
"Fuzz test has only be tested with build shared libs option using MSVC on windows")
|
|
cmake_args += [
|
|
"-Donnxruntime_BUILD_UNIT_TESTS=ON",
|
|
"-Donnxruntime_FUZZ_TEST=ON",
|
|
"-Donnxruntime_USE_FULL_PROTOBUF=ON"]
|
|
|
|
if args.gen_doc:
|
|
cmake_args += ["-Donnxruntime_PYBIND_EXPORT_OPSCHEMA=ON"]
|
|
else:
|
|
cmake_args += ["-Donnxruntime_PYBIND_EXPORT_OPSCHEMA=OFF"]
|
|
|
|
cmake_args += ["-D{}".format(define) for define in cmake_extra_defines]
|
|
|
|
cmake_args += cmake_extra_args
|
|
|
|
# ADO pipelines will store the pipeline build number
|
|
# (e.g. 191101-2300.1.master) and source version in environment
|
|
# variables. If present, use these values to define the
|
|
# WinML/ORT DLL versions.
|
|
build_number = os.getenv('Build_BuildNumber')
|
|
source_version = os.getenv('Build_SourceVersion')
|
|
if build_number and source_version:
|
|
build_matches = re.fullmatch(
|
|
r"(\d\d)(\d\d)(\d\d)(\d\d)\.(\d+)", build_number)
|
|
if build_matches:
|
|
YY = build_matches.group(2)
|
|
MM = build_matches.group(3)
|
|
DD = build_matches.group(4)
|
|
|
|
# Get ORT major and minor number
|
|
with open(os.path.join(source_dir, 'VERSION_NUMBER')) as f:
|
|
first_line = f.readline()
|
|
ort_version_matches = re.match(r"(\d+).(\d+)", first_line)
|
|
if not ort_version_matches:
|
|
raise BuildError("Couldn't read version from VERSION_FILE")
|
|
ort_major = ort_version_matches.group(1)
|
|
ort_minor = ort_version_matches.group(2)
|
|
# Example (BuildNumber: 191101-2300.1.master,
|
|
# SourceVersion: 0bce7ae6755c792eda558e5d27ded701707dc404)
|
|
# MajorPart = 1
|
|
# MinorPart = 0
|
|
# BuildPart = 1911
|
|
# PrivatePart = 123
|
|
# String = 191101-2300.1.master.0bce7ae
|
|
cmake_args += [
|
|
"-DVERSION_MAJOR_PART={}".format(ort_major),
|
|
"-DVERSION_MINOR_PART={}".format(ort_minor),
|
|
"-DVERSION_BUILD_PART={}".format(YY),
|
|
"-DVERSION_PRIVATE_PART={}{}".format(MM, DD),
|
|
"-DVERSION_STRING={}.{}.{}.{}".format(
|
|
ort_major, ort_minor, build_number,
|
|
source_version[0:7])
|
|
]
|
|
|
|
for config in configs:
|
|
config_build_dir = get_config_build_dir(build_dir, config)
|
|
os.makedirs(config_build_dir, exist_ok=True)
|
|
if args.use_nuphar:
|
|
os.environ["PATH"] = os.path.join(
|
|
config_build_dir, "external", "tvm",
|
|
config) + os.pathsep + os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"]
|
|
|
|
run_subprocess(
|
|
cmake_args + [
|
|
"-Donnxruntime_ENABLE_MEMLEAK_CHECKER=" +
|
|
("ON" if config.lower() == 'debug' and not args.use_nuphar and not
|
|
args.use_openvino and not
|
|
args.enable_msvc_static_runtime
|
|
else "OFF"), "-DCMAKE_BUILD_TYPE={}".format(config)],
|
|
cwd=config_build_dir)
|
|
|
|
|
|
def clean_targets(cmake_path, build_dir, configs):
|
|
for config in configs:
|
|
log.info("Cleaning targets for %s configuration", config)
|
|
build_dir2 = get_config_build_dir(build_dir, config)
|
|
cmd_args = [cmake_path,
|
|
"--build", build_dir2,
|
|
"--config", config,
|
|
"--target", "clean"]
|
|
|
|
run_subprocess(cmd_args)
|
|
|
|
|
|
def build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs, target=None):
|
|
for config in configs:
|
|
log.info("Building targets for %s configuration", config)
|
|
build_dir2 = get_config_build_dir(build_dir, config)
|
|
cmd_args = [cmake_path,
|
|
"--build", build_dir2,
|
|
"--config", config]
|
|
if target:
|
|
cmd_args.extend(['--target', target])
|
|
|
|
build_tool_args = []
|
|
if num_parallel_jobs != 1:
|
|
if is_windows() and args.cmake_generator != 'Ninja':
|
|
build_tool_args += [
|
|
"/maxcpucount:{}".format(num_parallel_jobs),
|
|
# if nodeReuse is true, msbuild processes will stay around for a bit after the build completes
|
|
"/nodeReuse:False",
|
|
]
|
|
elif (is_macOS() and args.use_xcode):
|
|
# CMake will generate correct build tool args for Xcode
|
|
cmd_args += ["--parallel", str(num_parallel_jobs)]
|
|
else:
|
|
build_tool_args += ["-j{}".format(num_parallel_jobs)]
|
|
|
|
if build_tool_args:
|
|
cmd_args += ["--"]
|
|
cmd_args += build_tool_args
|
|
|
|
env = {}
|
|
if args.android:
|
|
env['ANDROID_SDK_ROOT'] = args.android_sdk_path
|
|
|
|
run_subprocess(cmd_args, env=env)
|
|
|
|
|
|
def add_dir_if_exists(directory, dir_list):
|
|
if os.path.isdir(directory):
|
|
dir_list.append(directory)
|
|
|
|
|
|
def setup_cuda_vars(args):
|
|
cuda_home = ""
|
|
cudnn_home = ""
|
|
|
|
if args.use_cuda:
|
|
cuda_home = args.cuda_home if args.cuda_home else os.getenv(
|
|
"CUDA_HOME")
|
|
cudnn_home = args.cudnn_home if args.cudnn_home else os.getenv(
|
|
"CUDNN_HOME")
|
|
|
|
cuda_home_valid = (cuda_home is not None and os.path.exists(cuda_home))
|
|
cudnn_home_valid = (cudnn_home is not None and os.path.exists(
|
|
cudnn_home))
|
|
|
|
if not cuda_home_valid or not cudnn_home_valid:
|
|
raise BuildError(
|
|
"cuda_home and cudnn_home paths must be specified and valid.",
|
|
"cuda_home='{}' valid={}. cudnn_home='{}' valid={}"
|
|
.format(
|
|
cuda_home, cuda_home_valid, cudnn_home, cudnn_home_valid))
|
|
|
|
return cuda_home, cudnn_home
|
|
|
|
|
|
def setup_tensorrt_vars(args):
|
|
tensorrt_home = ""
|
|
if args.use_tensorrt:
|
|
tensorrt_home = (args.tensorrt_home if args.tensorrt_home
|
|
else os.getenv("TENSORRT_HOME"))
|
|
tensorrt_home_valid = (tensorrt_home is not None and
|
|
os.path.exists(tensorrt_home))
|
|
if not tensorrt_home_valid:
|
|
raise BuildError(
|
|
"tensorrt_home paths must be specified and valid.",
|
|
"tensorrt_home='{}' valid={}."
|
|
.format(tensorrt_home, tensorrt_home_valid))
|
|
|
|
# Set maximum workspace size in byte for
|
|
# TensorRT (1GB = 1073741824 bytes).
|
|
os.environ["ORT_TENSORRT_MAX_WORKSPACE_SIZE"] = "1073741824"
|
|
|
|
# Set maximum number of iterations to detect unsupported nodes
|
|
# and partition the models for TensorRT.
|
|
os.environ["ORT_TENSORRT_MAX_PARTITION_ITERATIONS"] = "1000"
|
|
|
|
# Set minimum subgraph node size in graph partitioning
|
|
# for TensorRT.
|
|
os.environ["ORT_TENSORRT_MIN_SUBGRAPH_SIZE"] = "1"
|
|
|
|
# Set FP16 flag
|
|
os.environ["ORT_TENSORRT_FP16_ENABLE"] = "0"
|
|
|
|
return tensorrt_home
|
|
|
|
|
|
def setup_migraphx_vars(args):
|
|
|
|
migraphx_home = None
|
|
|
|
if (args.use_migraphx):
|
|
print("migraphx_home = {}".format(args.migraphx_home))
|
|
migraphx_home = args.migraphx_home or os.getenv("MIGRAPHX_HOME") or None
|
|
|
|
migraphx_home_not_valid = (migraphx_home and not os.path.exists(migraphx_home))
|
|
|
|
if (migraphx_home_not_valid):
|
|
raise BuildError("migraphx_home paths must be specified and valid.",
|
|
"migraphx_home='{}' valid={}."
|
|
.format(migraphx_home, migraphx_home_not_valid))
|
|
return migraphx_home or ''
|
|
|
|
|
|
def setup_dml_build(args, cmake_path, build_dir, configs):
|
|
if args.use_dml:
|
|
for config in configs:
|
|
# Run the RESTORE_PACKAGES target to perform the initial
|
|
# NuGet setup.
|
|
cmd_args = [cmake_path,
|
|
"--build", get_config_build_dir(build_dir, config),
|
|
"--config", config,
|
|
"--target", "RESTORE_PACKAGES"]
|
|
run_subprocess(cmd_args)
|
|
|
|
|
|
def setup_rocm_build(args, configs):
|
|
|
|
rocm_home = None
|
|
|
|
if (args.use_rocm):
|
|
print("rocm_home = {}".format(args.rocm_home))
|
|
rocm_home = args.rocm_home or None
|
|
|
|
rocm_home_not_valid = (rocm_home and not os.path.exists(rocm_home))
|
|
|
|
if (rocm_home_not_valid):
|
|
raise BuildError("rocm_home paths must be specified and valid.",
|
|
"rocm_home='{}' valid={}."
|
|
.format(rocm_home, rocm_home_not_valid))
|
|
|
|
for config in configs:
|
|
amd_hipify(get_config_build_dir(args.build_dir, config))
|
|
return rocm_home or ''
|
|
|
|
|
|
def adb_push(src, dest, **kwargs):
|
|
return run_subprocess(['adb', 'push', src, dest], **kwargs)
|
|
|
|
|
|
def adb_shell(*args, **kwargs):
|
|
return run_subprocess(['adb', 'shell', *args], **kwargs)
|
|
|
|
|
|
def run_android_tests(args, source_dir, config, cwd):
|
|
|
|
def run_adb_shell(cmd):
|
|
# GCOV_PREFIX_STRIP specifies the depth of the directory hierarchy to stip and
|
|
# GCOV_PREFIX specifies the root directory
|
|
# for creating the runtime code coverage files.'
|
|
nonlocal cwd
|
|
if args.code_coverage:
|
|
adb_shell(
|
|
'cd /data/local/tmp && GCOV_PREFIX=/data/local/tmp \
|
|
GCOV_PREFIX_STRIP={} {}'.format(cwd.count(os.sep) + 1, cmd))
|
|
else:
|
|
adb_shell('cd /data/local/tmp && ' + cmd)
|
|
|
|
if args.android_abi == 'x86_64':
|
|
run_subprocess([os.path.join(
|
|
source_dir, 'tools', 'ci_build', 'github', 'android',
|
|
'start_android_emulator.sh')])
|
|
adb_push('testdata', '/data/local/tmp/', cwd=cwd)
|
|
adb_push(
|
|
os.path.join(source_dir, 'cmake', 'external', 'onnx', 'onnx', 'backend', 'test'),
|
|
'/data/local/tmp/', cwd=cwd)
|
|
adb_push('onnxruntime_test_all', '/data/local/tmp/', cwd=cwd)
|
|
adb_push('onnx_test_runner', '/data/local/tmp/', cwd=cwd)
|
|
run_adb_shell('/data/local/tmp/onnxruntime_test_all')
|
|
if args.use_nnapi:
|
|
adb_shell('cd /data/local/tmp && /data/local/tmp/onnx_test_runner -e nnapi /data/local/tmp/test')
|
|
else:
|
|
adb_shell('cd /data/local/tmp && /data/local/tmp/onnx_test_runner /data/local/tmp/test')
|
|
# run shared_lib_test if necessary
|
|
if args.build_shared_lib:
|
|
adb_push('libonnxruntime.so', '/data/local/tmp/', cwd=cwd)
|
|
adb_push('onnxruntime_shared_lib_test', '/data/local/tmp/', cwd=cwd)
|
|
run_adb_shell(
|
|
'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp && ' +
|
|
'/data/local/tmp/onnxruntime_shared_lib_test')
|
|
elif args.android_abi == 'arm64-v8a':
|
|
# For Android arm64 abi we are only verify the size of the binary generated by minimal build config
|
|
# Will fail the build if the shared_lib size is larger than the threshold
|
|
if args.minimal_build and config == 'MinSizeRel' and args.build_shared_lib and args.test_binary_size:
|
|
# set current size limit to 1165KB which is 110K large than 1.5.2 release.
|
|
bin_size_threshold = 1165000
|
|
bin_actual_size = os.path.getsize(os.path.join(cwd, 'libonnxruntime.so'))
|
|
log.info('Android arm64 minsizerel libonnxruntime.so size [' + str(bin_actual_size) + 'B]')
|
|
# Write the binary size to a file for uploading later
|
|
with open(os.path.join(cwd, 'binary_size_data.txt'), 'w') as file:
|
|
file.writelines([
|
|
'os,arch,build_config,size\n',
|
|
'android,arm64-v8a,minimal-baseline,' + str(bin_actual_size) + '\n'
|
|
])
|
|
if bin_actual_size > bin_size_threshold:
|
|
raise BuildError('Android arm64 minsizerel libonnxruntime.so size [' + str(bin_actual_size) +
|
|
'B] is bigger than threshold [' + str(bin_size_threshold) + 'B]')
|
|
|
|
|
|
def run_ios_tests(args, source_dir, config, cwd):
|
|
cpr = run_subprocess(["xcodebuild", "test", "-project", "./onnxruntime.xcodeproj",
|
|
"-configuration", config,
|
|
"-scheme", "onnxruntime_test_all_xc", "-destination",
|
|
"platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)"], cwd=cwd)
|
|
if cpr.returncode == 0:
|
|
cpr = run_subprocess(["xcodebuild", "test", "-project", "./onnxruntime.xcodeproj",
|
|
"-configuration", config,
|
|
"-scheme", "onnxruntime_shared_lib_test_xc", "-destination",
|
|
"platform=iOS Simulator,OS=latest,name=iPhone SE (2nd generation)"], cwd=cwd)
|
|
cpr.check_returncode()
|
|
|
|
|
|
def run_orttraining_test_orttrainer_frontend_separately(cwd):
|
|
class TestNameCollecterPlugin:
|
|
def __init__(self):
|
|
self.collected = set()
|
|
|
|
def pytest_collection_modifyitems(self, items):
|
|
for item in items:
|
|
print('item.name: ', item.name)
|
|
test_name = item.name
|
|
start = test_name.find('[')
|
|
if start > 0:
|
|
test_name = test_name[:start]
|
|
self.collected.add(test_name)
|
|
|
|
import pytest
|
|
|
|
plugin = TestNameCollecterPlugin()
|
|
test_script_filename = os.path.join(cwd, "orttraining_test_orttrainer_frontend.py")
|
|
pytest.main(['--collect-only', test_script_filename], plugins=[plugin])
|
|
|
|
for test_name in plugin.collected:
|
|
run_subprocess([
|
|
sys.executable, '-m', 'pytest',
|
|
'orttraining_test_orttrainer_frontend.py', '-v', '-k', test_name], cwd=cwd)
|
|
|
|
|
|
def run_training_python_frontend_tests(cwd):
|
|
run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer.py'], cwd=cwd)
|
|
run_subprocess([sys.executable, 'onnxruntime_test_training_unit_tests.py'], cwd=cwd)
|
|
run_subprocess([
|
|
sys.executable, 'orttraining_test_transformers.py',
|
|
'BertModelTest.test_for_pretraining_full_precision_list_input'], cwd=cwd)
|
|
run_subprocess([
|
|
sys.executable, 'orttraining_test_transformers.py',
|
|
'BertModelTest.test_for_pretraining_full_precision_dict_input'], cwd=cwd)
|
|
run_subprocess([
|
|
sys.executable, 'orttraining_test_transformers.py',
|
|
'BertModelTest.test_for_pretraining_full_precision_list_and_dict_input'], cwd=cwd)
|
|
|
|
# TODO: use run_orttraining_test_orttrainer_frontend_separately to work around a sporadic segfault.
|
|
# shall revert to run_subprocess call once the segfault issue is resolved.
|
|
run_orttraining_test_orttrainer_frontend_separately(cwd)
|
|
# run_subprocess([sys.executable, '-m', 'pytest', '-sv', 'orttraining_test_orttrainer_frontend.py'], cwd=cwd)
|
|
|
|
run_subprocess([sys.executable, '-m', 'pytest', '-sv', 'orttraining_test_orttrainer_bert_toy_onnx.py'], cwd=cwd)
|
|
|
|
|
|
def run_training_python_frontend_e2e_tests(cwd):
|
|
# frontend tests are to be added here:
|
|
log.info("Running python frontend e2e tests.")
|
|
|
|
run_subprocess(
|
|
[sys.executable, 'orttraining_run_frontend_batch_size_test.py', '-v'],
|
|
cwd=cwd, env={'CUDA_VISIBLE_DEVICES': '0'})
|
|
|
|
import torch
|
|
ngpus = torch.cuda.device_count()
|
|
if ngpus > 1:
|
|
bert_pretrain_script = 'orttraining_run_bert_pretrain.py'
|
|
# TODO: this test will be replaced with convergence test ported from backend
|
|
log.debug('RUN: mpirun -n {} ''-x' 'NCCL_DEBUG=INFO'' {} {} {}'.format(
|
|
ngpus, sys.executable, bert_pretrain_script, 'ORTBertPretrainTest.test_pretrain_convergence'))
|
|
run_subprocess([
|
|
'mpirun', '-n', str(ngpus), '-x', 'NCCL_DEBUG=INFO', sys.executable,
|
|
bert_pretrain_script, 'ORTBertPretrainTest.test_pretrain_convergence'], cwd=cwd)
|
|
|
|
log.debug('RUN: mpirun -n {} {} orttraining_run_glue.py'.format(ngpus, sys.executable))
|
|
run_subprocess([
|
|
'mpirun', '-n', str(ngpus), '-x', 'NCCL_DEBUG=INFO', sys.executable, 'orttraining_run_glue.py'], cwd=cwd)
|
|
|
|
# with orttraining_run_glue.py.
|
|
# 1. we like to force to use single GPU (with CUDA_VISIBLE_DEVICES)
|
|
# for fine-tune tests.
|
|
# 2. need to run test separately (not to mix between fp16
|
|
# and full precision runs. this need to be investigated).
|
|
run_subprocess(
|
|
[sys.executable, 'orttraining_run_glue.py', 'ORTGlueTest.test_bert_with_mrpc', '-v'],
|
|
cwd=cwd, env={'CUDA_VISIBLE_DEVICES': '0'})
|
|
|
|
run_subprocess(
|
|
[sys.executable, 'orttraining_run_glue.py', 'ORTGlueTest.test_bert_fp16_with_mrpc', '-v'],
|
|
cwd=cwd, env={'CUDA_VISIBLE_DEVICES': '0'})
|
|
|
|
run_subprocess(
|
|
[sys.executable, 'orttraining_run_glue.py', 'ORTGlueTest.test_roberta_with_mrpc', '-v'],
|
|
cwd=cwd, env={'CUDA_VISIBLE_DEVICES': '0'})
|
|
|
|
run_subprocess(
|
|
[sys.executable, 'orttraining_run_glue.py', 'ORTGlueTest.test_roberta_fp16_with_mrpc', '-v'],
|
|
cwd=cwd, env={'CUDA_VISIBLE_DEVICES': '0'})
|
|
|
|
run_subprocess(
|
|
[sys.executable, 'orttraining_run_multiple_choice.py', 'ORTMultipleChoiceTest.test_bert_fp16_with_swag', '-v'],
|
|
cwd=cwd, env={'CUDA_VISIBLE_DEVICES': '0'})
|
|
|
|
run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer_with_mixed_precision.py'], cwd=cwd)
|
|
|
|
run_subprocess([
|
|
sys.executable, 'orttraining_test_transformers.py',
|
|
'BertModelTest.test_for_pretraining_mixed_precision'], cwd=cwd)
|
|
|
|
# this test is not stable. it occasionally causes segfault due to its session creation/release pattern.
|
|
# need to skip to unblock release
|
|
# run_subprocess([
|
|
# sys.executable, 'orttraining_test_transformers.py',
|
|
# 'BertModelTest.test_for_pretraining_mixed_precision_with_gradient_accumulation'], cwd=cwd)
|
|
|
|
|
|
def run_training_pipeline_e2e_tests(cwd):
|
|
# pipeline tests are to be added here:
|
|
log.info("Running pipeline e2e tests.")
|
|
|
|
import torch
|
|
ngpus = torch.cuda.device_count()
|
|
|
|
command = ['./onnxruntime_training_bert',
|
|
'--ort_log_severity', '1',
|
|
'--optimizer=Lamb',
|
|
'--learning_rate=3e-3',
|
|
'--max_seq_length=128',
|
|
'--max_predictions_per_seq=20',
|
|
'--warmup_ratio=0.2843',
|
|
'--warmup_mode=Poly',
|
|
'--model_name', '/bert_ort/bert_models/nv/bert-large/' +
|
|
'bert-large-uncased_L_24_H_1024_A_16_V_30528_S_512_Dp_0.1_optimized_layer_norm_opset12',
|
|
'--train_data_dir', '/bert_data/128/books_wiki_en_corpus/train',
|
|
'--test_data_dir', '/bert_data/128/books_wiki_en_corpus/test',
|
|
'--display_loss_steps', '1',
|
|
'--use_nccl',
|
|
'--use_mixed_precision',
|
|
'--allreduce_in_fp16',
|
|
'--gradient_accumulation_steps', '48',
|
|
'--num_train_steps', '96',
|
|
'--train_batch_size', '50']
|
|
|
|
# TODO: currently the CI machine only has 4 GPUs for parallel tests.
|
|
# Fill in more pipeline partition options when the machine has different GPUs counts.
|
|
if ngpus != 4:
|
|
return
|
|
|
|
# Test 4-way pipeline parallel
|
|
pp_command = ['mpirun', '-n', str(ngpus)] + command + ['--pipeline_parallel_size', '4', '--cut_group_info',
|
|
'1149:407-1219/1341/1463/1585/1707/1829,' +
|
|
'1881:407-1951/2073/2195/2317/2439/2561,' +
|
|
'2613:407-2683/2805/2927/3049/3171/3293']
|
|
command_str = ', '.join(pp_command)
|
|
log.debug('RUN: ' + command_str)
|
|
run_subprocess(pp_command, cwd=cwd)
|
|
|
|
# Test 2-way data parallel + 2-way pipeline parallel
|
|
pp_dp_command = ['mpirun', '-n', str(ngpus)]
|
|
pp_dp_command = pp_dp_command + command
|
|
pp_dp_command = pp_dp_command + ['--data_parallel_size', '2', '--pipeline_parallel_size',
|
|
'2', '--cut_group_info',
|
|
'1881:407-1951/2073/2195/2317/2439/2561/2683/2805/2927/3049/3171/3293']
|
|
command_str = ', '.join(pp_dp_command)
|
|
log.debug('RUN: ' + command_str)
|
|
run_subprocess(pp_dp_command, cwd=cwd)
|
|
|
|
|
|
def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
|
|
for config in configs:
|
|
log.info("Running tests for %s configuration", config)
|
|
cwd = get_config_build_dir(build_dir, config)
|
|
cwd = os.path.abspath(cwd)
|
|
|
|
# TODO: temporarily disable this test to restore pipeline health. This test fails due to
|
|
# an OOM regression. Invetigation undergoing.
|
|
# if args.enable_training and args.use_cuda and args.enable_training_pipeline_e2e_tests:
|
|
# # run distributed pipeline test on 4-GPU CI machine.
|
|
# run_training_pipeline_e2e_tests(cwd=cwd)
|
|
# continue
|
|
|
|
if args.android:
|
|
run_android_tests(args, source_dir, config, cwd)
|
|
continue
|
|
elif args.ios:
|
|
run_ios_tests(args, source_dir, config, cwd)
|
|
continue
|
|
dll_path_list = []
|
|
if args.use_nuphar:
|
|
dll_path_list.append(os.path.join(
|
|
build_dir, config, "external", "tvm", config))
|
|
if args.use_tensorrt:
|
|
dll_path_list.append(os.path.join(args.tensorrt_home, 'lib'))
|
|
if args.use_mklml:
|
|
dll_path_list.append(os.path.join(build_dir, config, "mklml", "src", "project_mklml", "lib"))
|
|
|
|
dll_path = None
|
|
if len(dll_path_list) > 0:
|
|
dll_path = os.pathsep.join(dll_path_list)
|
|
|
|
if ctest_path is None:
|
|
# Get the "Google Test Adapter" for vstest.
|
|
if not os.path.exists(os.path.join(cwd,
|
|
'googletestadapter.0.17.1')):
|
|
run_subprocess(
|
|
['nuget.exe', 'restore',
|
|
os.path.join(source_dir, 'packages.config'),
|
|
'-ConfigFile', os.path.join(source_dir, 'NuGet.config'),
|
|
'-PackagesDirectory', cwd])
|
|
cwd2 = os.path.join(cwd, config)
|
|
executables = ['onnxruntime_test_all.exe']
|
|
if args.build_shared_lib:
|
|
executables.append('onnxruntime_shared_lib_test.exe')
|
|
executables.append('onnxruntime_global_thread_pools_test.exe')
|
|
run_subprocess(
|
|
['vstest.console.exe', '--parallel',
|
|
'--TestAdapterPath:..\\googletestadapter.0.17.1\\build\\_common', # noqa
|
|
'/Logger:trx', '/Enablecodecoverage', '/Platform:x64',
|
|
"/Settings:%s" % os.path.join(
|
|
source_dir, 'cmake\\codeconv.runsettings')] + executables,
|
|
cwd=cwd2, dll_path=dll_path)
|
|
else:
|
|
ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "3600"]
|
|
run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path)
|
|
|
|
if args.enable_pybind:
|
|
# Disable python tests for TensorRT because many tests are
|
|
# not supported yet.
|
|
if args.use_tensorrt:
|
|
return
|
|
|
|
# Disable python tests in a reduced build as we don't know which ops have been included and which
|
|
# models can run
|
|
if args.include_ops_by_model or args.include_ops_by_config or args.minimal_build != 'off':
|
|
return
|
|
|
|
if is_windows():
|
|
cwd = os.path.join(cwd, config)
|
|
|
|
run_subprocess([sys.executable, 'onnxruntime_test_python.py'], cwd=cwd, dll_path=dll_path)
|
|
|
|
if args.enable_symbolic_shape_infer_tests:
|
|
run_subprocess([sys.executable, 'onnxruntime_test_python_symbolic_shape_infer.py'],
|
|
cwd=cwd, dll_path=dll_path)
|
|
|
|
# For CUDA enabled builds test IOBinding feature
|
|
if args.use_cuda:
|
|
# We need to have Torch installed to test the IOBinding feature
|
|
# which currently uses Torch's allocator to allocate GPU memory for testing
|
|
log.info("Testing IOBinding feature")
|
|
run_subprocess([sys.executable, 'onnxruntime_test_python_iobinding.py'], cwd=cwd, dll_path=dll_path)
|
|
|
|
if not args.disable_ml_ops:
|
|
run_subprocess([sys.executable, 'onnxruntime_test_python_mlops.py'], cwd=cwd, dll_path=dll_path)
|
|
|
|
if args.enable_training and args.use_cuda:
|
|
# run basic frontend tests
|
|
run_training_python_frontend_tests(cwd=cwd)
|
|
|
|
try:
|
|
import onnx # noqa
|
|
onnx_test = True
|
|
except ImportError as error:
|
|
log.exception(error)
|
|
log.warning("onnx is not installed. The ONNX tests will be skipped.")
|
|
onnx_test = False
|
|
|
|
if onnx_test:
|
|
run_subprocess([sys.executable, 'onnxruntime_test_python_backend.py'], cwd=cwd, dll_path=dll_path)
|
|
|
|
if not args.disable_ml_ops:
|
|
run_subprocess([sys.executable, 'onnxruntime_test_python_backend_mlops.py'],
|
|
cwd=cwd, dll_path=dll_path)
|
|
|
|
run_subprocess([sys.executable,
|
|
os.path.join(source_dir, 'onnxruntime', 'test', 'onnx', 'gen_test_models.py'),
|
|
'--output_dir', 'test_models'], cwd=cwd)
|
|
|
|
if not args.skip_onnx_tests:
|
|
run_subprocess([os.path.join(cwd, 'onnx_test_runner'), 'test_models'], cwd=cwd)
|
|
if config != 'Debug':
|
|
run_subprocess([sys.executable, 'onnx_backend_test_series.py'], cwd=cwd, dll_path=dll_path)
|
|
|
|
if not args.skip_keras_test:
|
|
try:
|
|
import onnxmltools # noqa
|
|
import keras # noqa
|
|
onnxml_test = True
|
|
except ImportError:
|
|
log.warning(
|
|
"onnxmltools and keras are not installed. "
|
|
"The keras tests will be skipped.")
|
|
onnxml_test = False
|
|
if onnxml_test:
|
|
run_subprocess(
|
|
[sys.executable, 'onnxruntime_test_python_keras.py'],
|
|
cwd=cwd, dll_path=dll_path)
|
|
|
|
|
|
def nuphar_run_python_tests(build_dir, configs):
|
|
"""nuphar temporary function for running python tests separately
|
|
as it requires ONNX 1.5.0
|
|
"""
|
|
for config in configs:
|
|
if config == 'Debug':
|
|
continue
|
|
cwd = get_config_build_dir(build_dir, config)
|
|
if is_windows():
|
|
cwd = os.path.join(cwd, config)
|
|
dll_path = os.path.join(build_dir, config, "external", "tvm", config)
|
|
# install onnx for shape inference in testing Nuphar scripts
|
|
# this needs to happen after onnx_test_data preparation which
|
|
# uses onnx 1.3.0
|
|
run_subprocess(
|
|
[sys.executable, '-m', 'pip', 'install', '--user', 'onnx==1.5.0'])
|
|
run_subprocess(
|
|
[sys.executable, 'onnxruntime_test_python_nuphar.py'],
|
|
cwd=cwd, dll_path=dll_path)
|
|
|
|
|
|
def run_nodejs_tests(nodejs_binding_dir):
|
|
args = ['npm', 'test', '--', '--timeout=2000']
|
|
if is_windows():
|
|
args = ['cmd', '/c'] + args
|
|
run_subprocess(args, cwd=nodejs_binding_dir)
|
|
|
|
|
|
def build_python_wheel(
|
|
source_dir, build_dir, configs, use_cuda, use_dnnl,
|
|
use_tensorrt, use_openvino, use_nuphar, use_vitisai, use_acl, use_armnn, use_dml,
|
|
wheel_name_suffix, enable_training, nightly_build=False, featurizers_build=False, use_ninja=False):
|
|
for config in configs:
|
|
cwd = get_config_build_dir(build_dir, config)
|
|
if is_windows() and not use_ninja:
|
|
cwd = os.path.join(cwd, config)
|
|
|
|
args = [sys.executable, os.path.join(source_dir, 'setup.py'),
|
|
'bdist_wheel']
|
|
|
|
# We explicitly override the platform tag in the name of the generated build wheel
|
|
# so that we can install the wheel on Mac OS X versions 10.12+.
|
|
# Without this explicit override, we will something like this while building on MacOS 10.14 -
|
|
# [WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value (10.12)
|
|
# than the version on which the Python interpreter was compiled (10.14) and will be ignored.
|
|
# Since we need to support 10.12+, we explicitly override the platform tag.
|
|
# See PR #3626 for more details
|
|
if is_macOS():
|
|
args += ['-p', 'macosx_10_12_x86_64']
|
|
|
|
# Any combination of the following arguments can be applied
|
|
if nightly_build:
|
|
args.append('--nightly_build')
|
|
if featurizers_build:
|
|
args.append("--use_featurizers")
|
|
if wheel_name_suffix:
|
|
args.append('--wheel_name_suffix={}'.format(wheel_name_suffix))
|
|
if enable_training:
|
|
args.append("--enable_training")
|
|
|
|
# The following arguments are mutually exclusive
|
|
if use_tensorrt:
|
|
args.append('--use_tensorrt')
|
|
elif use_cuda:
|
|
args.append('--use_cuda')
|
|
elif use_openvino:
|
|
args.append('--use_openvino')
|
|
elif use_dnnl:
|
|
args.append('--use_dnnl')
|
|
elif use_nuphar:
|
|
args.append('--use_nuphar')
|
|
elif use_vitisai:
|
|
args.append('--use_vitisai')
|
|
elif use_acl:
|
|
args.append('--use_acl')
|
|
elif use_armnn:
|
|
args.append('--use_armnn')
|
|
elif use_dml:
|
|
args.append('--use_dml')
|
|
|
|
run_subprocess(args, cwd=cwd)
|
|
|
|
|
|
def derive_linux_build_property():
|
|
if is_windows():
|
|
return "/p:IsLinuxBuild=\"false\""
|
|
else:
|
|
return "/p:IsLinuxBuild=\"true\""
|
|
|
|
|
|
def build_nuget_package(source_dir, build_dir, configs, use_cuda, use_openvino, use_tensorrt, use_dnnl, use_mklml):
|
|
if not (is_windows() or is_linux()):
|
|
raise BuildError(
|
|
'Currently csharp builds and nuget package creation is only supportted '
|
|
'on Windows and Linux platforms.')
|
|
|
|
csharp_build_dir = os.path.join(source_dir, 'csharp')
|
|
is_linux_build = derive_linux_build_property()
|
|
|
|
# derive package name and execution provider based on the build args
|
|
execution_provider = "/p:ExecutionProvider=\"None\""
|
|
package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime\""
|
|
if use_openvino:
|
|
execution_provider = "/p:ExecutionProvider=\"openvino\""
|
|
package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.OpenVino\""
|
|
elif use_tensorrt:
|
|
execution_provider = "/p:ExecutionProvider=\"tensorrt\""
|
|
package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.TensorRT\""
|
|
elif use_dnnl:
|
|
execution_provider = "/p:ExecutionProvider=\"dnnl\""
|
|
package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.DNNL\""
|
|
elif use_cuda:
|
|
package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.Gpu\""
|
|
elif use_mklml:
|
|
package_name = "/p:OrtPackageId=\"Microsoft.ML.OnnxRuntime.MKLML\""
|
|
else:
|
|
pass
|
|
|
|
# set build directory based on build_dir arg
|
|
native_dir = os.path.normpath(os.path.join(source_dir, build_dir))
|
|
ort_build_dir = "/p:OnnxRuntimeBuildDirectory=\"" + native_dir + "\""
|
|
|
|
# dotnet restore
|
|
cmd_args = ["dotnet", "restore", "OnnxRuntime.CSharp.sln", "--configfile", "Nuget.CSharp.config"]
|
|
run_subprocess(cmd_args, cwd=csharp_build_dir)
|
|
|
|
# build csharp bindings and create nuget package for each config
|
|
for config in configs:
|
|
if is_linux():
|
|
native_build_dir = os.path.join(native_dir, config)
|
|
cmd_args = ["make", "install", "DESTDIR=.//nuget-staging"]
|
|
run_subprocess(cmd_args, cwd=native_build_dir)
|
|
|
|
configuration = "/p:Configuration=\"" + config + "\""
|
|
|
|
cmd_args = ["dotnet", "msbuild", "OnnxRuntime.CSharp.sln", configuration, package_name, is_linux_build,
|
|
ort_build_dir]
|
|
run_subprocess(cmd_args, cwd=csharp_build_dir)
|
|
|
|
cmd_args = [
|
|
"dotnet", "msbuild", "OnnxRuntime.CSharp.proj", "/t:CreatePackage",
|
|
package_name, configuration, execution_provider, is_linux_build, ort_build_dir]
|
|
run_subprocess(cmd_args, cwd=csharp_build_dir)
|
|
|
|
|
|
def run_csharp_tests(source_dir, build_dir, use_cuda, use_openvino, use_tensorrt, use_dnnl):
|
|
# Currently only running tests on windows.
|
|
if not is_windows():
|
|
return
|
|
csharp_source_dir = os.path.join(source_dir, 'csharp')
|
|
is_linux_build = derive_linux_build_property()
|
|
|
|
# define macros based on build args
|
|
macros = ""
|
|
if use_openvino:
|
|
macros += "USE_OPENVINO;"
|
|
if use_tensorrt:
|
|
macros += "USE_TENSORRT;"
|
|
if use_dnnl:
|
|
macros += "USE_DNNL;"
|
|
if use_cuda:
|
|
macros += "USE_CUDA;"
|
|
|
|
define_constants = ""
|
|
if macros != "":
|
|
define_constants = "/p:DefineConstants=\"" + macros + "\""
|
|
|
|
# set build directory based on build_dir arg
|
|
native_build_dir = os.path.normpath(os.path.join(source_dir, build_dir))
|
|
ort_build_dir = "/p:OnnxRuntimeBuildDirectory=\"" + native_build_dir + "\""
|
|
|
|
# Skip pretrained models test. Only run unit tests as part of the build
|
|
# add "--verbosity", "detailed" to this command if required
|
|
cmd_args = ["dotnet", "test", "test\\Microsoft.ML.OnnxRuntime.Tests\\Microsoft.ML.OnnxRuntime.Tests.csproj",
|
|
"--filter", "FullyQualifiedName!=Microsoft.ML.OnnxRuntime.Tests.InferenceTest.TestPreTrainedModels",
|
|
is_linux_build, define_constants, ort_build_dir]
|
|
run_subprocess(cmd_args, cwd=csharp_source_dir)
|
|
|
|
|
|
def is_cross_compiling_on_apple(args):
|
|
if not is_macOS():
|
|
return False
|
|
if args.ios:
|
|
return True
|
|
if args.osx_arch != platform.machine():
|
|
return True
|
|
return False
|
|
|
|
|
|
def build_protoc_for_host(cmake_path, source_dir, build_dir, args):
|
|
if (args.arm or args.arm64 or args.enable_windows_store) and \
|
|
not (is_windows() or is_cross_compiling_on_apple(args)):
|
|
raise BuildError(
|
|
'Currently only support building protoc for Windows host while '
|
|
'cross-compiling for ARM/ARM64/Store and linux cross-compiling iOS')
|
|
|
|
log.info(
|
|
"Building protoc for host to be used in cross-compiled build process")
|
|
protoc_build_dir = os.path.join(os.getcwd(), build_dir, 'host_protoc')
|
|
os.makedirs(protoc_build_dir, exist_ok=True)
|
|
# Generate step
|
|
cmd_args = [
|
|
cmake_path,
|
|
os.path.join(source_dir, 'cmake', 'external', 'protobuf', 'cmake'),
|
|
'-Dprotobuf_BUILD_TESTS=OFF',
|
|
'-Dprotobuf_WITH_ZLIB_DEFAULT=OFF',
|
|
'-Dprotobuf_BUILD_SHARED_LIBS=OFF'
|
|
]
|
|
|
|
is_ninja = args.cmake_generator == 'Ninja'
|
|
if args.cmake_generator is not None and not (is_macOS() and args.use_xcode):
|
|
cmd_args += ['-G', args.cmake_generator]
|
|
if is_windows():
|
|
if not is_ninja:
|
|
cmd_args += ['-T', 'host=x64']
|
|
elif is_macOS():
|
|
if args.use_xcode:
|
|
cmd_args += ['-G', 'Xcode']
|
|
# CMake < 3.18 has a bug setting system arch to arm64 (if not specified) for Xcode 12,
|
|
# protoc for host should be built using host architecture
|
|
# Explicitly specify the CMAKE_OSX_ARCHITECTURES for x86_64 Mac.
|
|
cmd_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(
|
|
'arm64' if platform.machine() == 'arm64' else 'x86_64')]
|
|
|
|
run_subprocess(cmd_args, cwd=protoc_build_dir)
|
|
# Build step
|
|
cmd_args = [cmake_path,
|
|
"--build", protoc_build_dir,
|
|
"--config", "Release",
|
|
"--target", "protoc"]
|
|
run_subprocess(cmd_args)
|
|
|
|
# Absolute protoc path is needed for cmake
|
|
config_dir = ''
|
|
suffix = ''
|
|
|
|
if (is_windows() and not is_ninja) or (is_macOS() and args.use_xcode):
|
|
config_dir = 'Release'
|
|
|
|
if is_windows():
|
|
suffix = '.exe'
|
|
|
|
expected_protoc_path = os.path.join(protoc_build_dir, config_dir, 'protoc' + suffix)
|
|
|
|
if not os.path.exists(expected_protoc_path):
|
|
raise BuildError("Couldn't find {}. Host build of protoc failed.".format(expected_protoc_path))
|
|
|
|
return expected_protoc_path
|
|
|
|
|
|
def generate_documentation(source_dir, build_dir, configs):
|
|
operator_doc_path = os.path.join(source_dir, 'docs', 'ContribOperators.md')
|
|
opkernel_doc_path = os.path.join(source_dir, 'docs', 'OperatorKernels.md')
|
|
for config in configs:
|
|
# Copy the gen_contrib_doc.py.
|
|
shutil.copy(
|
|
os.path.join(source_dir, 'tools', 'python', 'gen_contrib_doc.py'),
|
|
os.path.join(build_dir, config))
|
|
shutil.copy(
|
|
os.path.join(source_dir, 'tools', 'python', 'gen_opkernel_doc.py'),
|
|
os.path.join(build_dir, config))
|
|
run_subprocess(
|
|
[sys.executable,
|
|
'gen_contrib_doc.py',
|
|
'--output_path', operator_doc_path],
|
|
cwd=os.path.join(build_dir, config))
|
|
run_subprocess(
|
|
[sys.executable,
|
|
'gen_opkernel_doc.py',
|
|
'--output_path', opkernel_doc_path],
|
|
cwd=os.path.join(build_dir, config))
|
|
docdiff = ''
|
|
try:
|
|
docdiff = subprocess.check_output(['git', 'diff', opkernel_doc_path])
|
|
except subprocess.CalledProcessError:
|
|
print('git diff returned non-zero error code')
|
|
if len(docdiff) > 0:
|
|
# Show warning instead of throwing exception, because it is
|
|
# dependent on build configuration for including
|
|
# execution propviders
|
|
log.warning(
|
|
'The updated opkernel document file ' + str(opkernel_doc_path) +
|
|
' is different from the checked in version. Consider '
|
|
'regenerating the file with CPU, DNNL and CUDA providers enabled.')
|
|
log.debug('diff:\n' + str(docdiff))
|
|
|
|
docdiff = ''
|
|
try:
|
|
docdiff = subprocess.check_output(['git', 'diff', operator_doc_path])
|
|
except subprocess.CalledProcessError:
|
|
print('git diff returned non-zero error code')
|
|
if len(docdiff) > 0:
|
|
raise BuildError(
|
|
'The updated operator document file ' +
|
|
str(operator_doc_path) + ' must be checked in.\n diff:\n' +
|
|
str(docdiff))
|
|
|
|
|
|
def main():
|
|
args = parse_arguments()
|
|
cmake_extra_defines = (args.cmake_extra_defines
|
|
if args.cmake_extra_defines else [])
|
|
cross_compiling = args.arm or args.arm64 or args.android
|
|
|
|
# If there was no explicit argument saying what to do, default
|
|
# to update, build and test (for native builds).
|
|
if not (args.update or args.clean or args.build or args.test):
|
|
log.debug(
|
|
"Defaulting to running update, build "
|
|
"[and test for native builds].")
|
|
args.update = True
|
|
args.build = True
|
|
if cross_compiling:
|
|
args.test = args.android_abi == 'x86_64' or args.android_abi == 'arm64-v8a'
|
|
else:
|
|
args.test = True
|
|
|
|
if args.skip_tests:
|
|
args.test = False
|
|
|
|
if args.include_ops_by_model or args.include_ops_by_config:
|
|
from exclude_unused_ops import exclude_unused_ops
|
|
models_path = args.include_ops_by_model if args.include_ops_by_model else ''
|
|
config_path = args.include_ops_by_config if args.include_ops_by_config else ''
|
|
exclude_unused_ops(models_path, config_path, use_cuda=args.use_cuda)
|
|
|
|
if args.use_tensorrt:
|
|
args.use_cuda = True
|
|
|
|
if args.build_wheel or args.gen_doc:
|
|
args.enable_pybind = True
|
|
|
|
if args.build_csharp or args.build_nuget or args.build_java or args.build_nodejs:
|
|
args.build_shared_lib = True
|
|
|
|
if args.build_nuget and cross_compiling:
|
|
raise BuildError('Currently nuget package creation is not supported while cross-compiling')
|
|
|
|
if args.enable_pybind and args.disable_exceptions:
|
|
raise BuildError('Python bindings require exceptions to be enabled.')
|
|
|
|
if args.minimal_build and args.disable_ort_format_load:
|
|
raise BuildError('Minimal build requires loading ORT format models.')
|
|
|
|
if args.nnapi_min_api:
|
|
if not args.use_nnapi:
|
|
raise BuildError("Using --nnapi_min_api requires --use_nnapi")
|
|
if args.nnapi_min_api < 27:
|
|
raise BuildError("--nnapi_min_api should be 27+")
|
|
|
|
if args.code_coverage and not args.android:
|
|
raise BuildError("Using --code_coverage requires --android")
|
|
|
|
# Disabling unit tests for VAD-F as FPGA only supports
|
|
# models with NCHW layout
|
|
if args.use_openvino == "VAD-F_FP32":
|
|
args.test = False
|
|
|
|
configs = set(args.config)
|
|
|
|
# setup paths and directories
|
|
cmake_path = resolve_executable_path(args.cmake_path)
|
|
ctest_path = None if args.use_vstest else resolve_executable_path(
|
|
args.ctest_path)
|
|
build_dir = args.build_dir
|
|
script_dir = os.path.realpath(os.path.dirname(__file__))
|
|
source_dir = os.path.normpath(os.path.join(script_dir, "..", ".."))
|
|
|
|
# if using cuda, setup cuda paths and env vars
|
|
cuda_home, cudnn_home = setup_cuda_vars(args)
|
|
|
|
mpi_home = args.mpi_home
|
|
nccl_home = args.nccl_home
|
|
|
|
acl_home = args.acl_home
|
|
acl_libs = args.acl_libs
|
|
|
|
armnn_home = args.armnn_home
|
|
armnn_libs = args.armnn_libs
|
|
|
|
# if using tensorrt, setup tensorrt paths
|
|
tensorrt_home = setup_tensorrt_vars(args)
|
|
|
|
# if using migraphx, setup migraphx paths
|
|
migraphx_home = setup_migraphx_vars(args)
|
|
|
|
# if using rocm, setup rocm paths
|
|
rocm_home = setup_rocm_build(args, configs)
|
|
|
|
os.makedirs(build_dir, exist_ok=True)
|
|
|
|
log.info("Build started")
|
|
if args.update:
|
|
cmake_extra_args = []
|
|
path_to_protoc_exe = args.path_to_protoc_exe
|
|
if not args.skip_submodule_sync:
|
|
update_submodules(source_dir)
|
|
if is_windows():
|
|
if args.cmake_generator == 'Ninja':
|
|
if args.x86 or args.arm or args.arm64:
|
|
raise BuildError(
|
|
"To cross-compile with Ninja, load the toolset "
|
|
"environment for the target processor (e.g. Cross "
|
|
"Tools Command Prompt for VS)")
|
|
cmake_extra_args = ['-G', args.cmake_generator]
|
|
elif args.x86:
|
|
cmake_extra_args = [
|
|
'-A', 'Win32', '-T', 'host=x64', '-G', args.cmake_generator
|
|
]
|
|
elif args.arm or args.arm64:
|
|
# Cross-compiling for ARM(64) architecture
|
|
# First build protoc for host to use during cross-compilation
|
|
if path_to_protoc_exe is None:
|
|
path_to_protoc_exe = build_protoc_for_host(
|
|
cmake_path, source_dir, build_dir, args)
|
|
if args.arm:
|
|
cmake_extra_args = ['-A', 'ARM']
|
|
else:
|
|
cmake_extra_args = ['-A', 'ARM64']
|
|
cmake_extra_args += ['-G', args.cmake_generator]
|
|
# Cannot test on host build machine for cross-compiled
|
|
# builds (Override any user-defined behaviour for test if any)
|
|
if args.test:
|
|
log.warning(
|
|
"Cannot test on host build machine for cross-compiled "
|
|
"ARM(64) builds. Will skip test running after build.")
|
|
args.test = False
|
|
else:
|
|
if (args.msvc_toolset == '14.16' and
|
|
args.cmake_generator == 'Visual Studio 16 2019'):
|
|
# CUDA 10.0 requires _MSC_VER >= 1700 and
|
|
# _MSC_VER < 1920, aka Visual Studio version
|
|
# in [2012, 2019). In VS2019, we have to use
|
|
# Side-by-side minor version MSVC toolsets from
|
|
# Visual Studio 2017 14.16 is MSVC version
|
|
# 141 is MSVC Toolset Version
|
|
# Cuda VS extension should be installed to
|
|
# C:\Program Files (x86)\Microsoft Visual
|
|
# Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations # noqa
|
|
toolset = 'v141,host=x64,version=' + args.msvc_toolset
|
|
elif args.msvc_toolset:
|
|
toolset = 'host=x64,version=' + args.msvc_toolset
|
|
else:
|
|
toolset = 'host=x64'
|
|
if args.cuda_version:
|
|
toolset += ',cuda=' + args.cuda_version
|
|
cmake_extra_args = [
|
|
'-A', 'x64', '-T', toolset, '-G', args.cmake_generator
|
|
]
|
|
if args.enable_windows_store:
|
|
cmake_extra_args.append(
|
|
'-DCMAKE_TOOLCHAIN_FILE=' + os.path.join(
|
|
source_dir, 'cmake', 'store_toolchain.cmake'))
|
|
if args.enable_wcos:
|
|
cmake_extra_args.append('-DCMAKE_USER_MAKE_RULES_OVERRIDE=wcos_rules_override.cmake')
|
|
elif args.cmake_generator is not None and not (is_macOS() and args.use_xcode):
|
|
cmake_extra_args += ['-G', args.cmake_generator]
|
|
elif is_macOS():
|
|
if args.use_xcode:
|
|
cmake_extra_args += ['-G', 'Xcode']
|
|
if not args.ios and not args.android and \
|
|
args.osx_arch == 'arm64' and platform.machine() == 'x86_64':
|
|
if args.test:
|
|
log.warning(
|
|
"Cannot test ARM64 build on X86_64. Will skip test running after build.")
|
|
args.test = False
|
|
|
|
if (args.android or args.ios or args.enable_windows_store
|
|
or is_cross_compiling_on_apple(args)) and args.path_to_protoc_exe is None:
|
|
# Cross-compiling for Android and iOS
|
|
path_to_protoc_exe = build_protoc_for_host(
|
|
cmake_path, source_dir, build_dir, args)
|
|
|
|
if is_ubuntu_1604():
|
|
if (args.arm or args.arm64):
|
|
raise BuildError(
|
|
"Only Windows ARM(64) cross-compiled builds supported "
|
|
"currently through this script")
|
|
install_ubuntu_deps(args)
|
|
if not is_docker() and not args.use_acl and not args.use_armnn:
|
|
install_python_deps()
|
|
if args.enable_pybind and is_windows():
|
|
install_python_deps(args.numpy_version)
|
|
if args.enable_onnx_tests:
|
|
setup_test_data(build_dir, configs)
|
|
generate_build_tree(
|
|
cmake_path, source_dir, build_dir, cuda_home, cudnn_home, rocm_home, mpi_home, nccl_home,
|
|
tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs,
|
|
path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args)
|
|
|
|
if args.clean:
|
|
clean_targets(cmake_path, build_dir, configs)
|
|
|
|
# if using DML, perform initial nuget package restore
|
|
setup_dml_build(args, cmake_path, build_dir, configs)
|
|
|
|
if args.build:
|
|
if args.parallel < 0:
|
|
raise BuildError("Invalid parallel job count: {}".format(args.parallel))
|
|
num_parallel_jobs = os.cpu_count() if args.parallel == 0 else args.parallel
|
|
build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs, args.target)
|
|
|
|
if args.test:
|
|
run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs)
|
|
|
|
# run nuphar python tests last, as it installs ONNX 1.5.0
|
|
if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar:
|
|
nuphar_run_python_tests(build_dir, configs)
|
|
|
|
# run node.js binding tests
|
|
if args.build_nodejs and not args.skip_nodejs_tests:
|
|
nodejs_binding_dir = os.path.normpath(os.path.join(source_dir, "nodejs"))
|
|
run_nodejs_tests(nodejs_binding_dir)
|
|
|
|
if args.build:
|
|
if args.build_wheel:
|
|
nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1')
|
|
build_python_wheel(
|
|
source_dir,
|
|
build_dir,
|
|
configs,
|
|
args.use_cuda,
|
|
args.use_dnnl,
|
|
args.use_tensorrt,
|
|
args.use_openvino,
|
|
args.use_nuphar,
|
|
args.use_vitisai,
|
|
args.use_acl,
|
|
args.use_armnn,
|
|
args.use_dml,
|
|
args.wheel_name_suffix,
|
|
args.enable_training,
|
|
nightly_build=nightly_build,
|
|
featurizers_build=args.use_featurizers,
|
|
use_ninja=(args.cmake_generator == 'Ninja')
|
|
)
|
|
if args.build_nuget:
|
|
build_nuget_package(
|
|
source_dir,
|
|
build_dir,
|
|
configs,
|
|
args.use_cuda,
|
|
args.use_openvino,
|
|
args.use_tensorrt,
|
|
args.use_dnnl,
|
|
args.use_mklml
|
|
)
|
|
|
|
if args.test and args.build_nuget:
|
|
run_csharp_tests(
|
|
source_dir,
|
|
build_dir,
|
|
args.use_cuda,
|
|
args.use_openvino,
|
|
args.use_tensorrt,
|
|
args.use_dnnl)
|
|
|
|
if args.gen_doc and (args.build or args.test):
|
|
generate_documentation(source_dir, build_dir, configs)
|
|
|
|
log.info("Build complete")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except BaseError as e:
|
|
log.error(str(e))
|
|
sys.exit(1)
|