diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 711a9d1f33..0b2f980ab2 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -2,8 +2,14 @@ # Licensed under the MIT License. # Minimum CMake required -# TODO: actually we need cmake 3.26+ on Windows but QNN EP's build machine doesn't have it -cmake_minimum_required(VERSION 3.24) +if(WIN32) + # TODO: actually we need cmake 3.26+ on Windows but QNN EP's build machine doesn't have it + cmake_minimum_required(VERSION 3.25) +elseif(onnxruntime_USE_ROCM) + cmake_minimum_required(VERSION 3.24) +else() + cmake_minimum_required(VERSION 3.26) +endif() cmake_policy(SET CMP0069 NEW) set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) diff --git a/cmake/deps.txt b/cmake/deps.txt index d0a5cf7aa4..6b7fb0c95f 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -27,6 +27,11 @@ onnx;https://github.com/onnx/onnx/archive/3b58938e025c41d2fcd89fa22028eefaa81a18 #use the last commit of 8.6-EA branch (https://github.com/onnx/onnx-tensorrt/commit/ba6a4fb34fdeaa3613bf981610c657e7b663a699) onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/ba6a4fb34fdeaa3613bf981610c657e7b663a699.zip;5a474ed86e2c4ee4085d3daeff8222866e933dc0 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa +protoc_win64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip;b4521f7ada5b260380f94c4bd7f1b7684c76969a +protoc_win32;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win32.zip;3688010318192c46ce73213cdfb6b3e5656da874 +protoc_linux_x64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-x86_64.zip;338462004aa5be9fba45b35b5b4be43f69b47a90 +protoc_linux_x86;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-x86_32.zip;61fdbe7d6360e065ec6fea23bca2cca673115fb8 +protoc_linux_aarch64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-linux-aarch_64.zip;df9d45470b0b8cf939dd2f0ec6b88e9cafc4d617 psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013 pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101 pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13 diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 0c0dea7df2..9effd1a2db 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -100,6 +100,44 @@ FetchContent_Declare( FIND_PACKAGE_ARGS 1.12.0...<2.0.0 NAMES Flatbuffers ) +# Download a protoc binary from Internet if needed +if(CMAKE_CROSSCOMPILING AND NOT ONNX_CUSTOM_PROTOC_EXECUTABLE AND NOT CMAKE_OSX_ARCHITECTURES) + # This part of code is only for users' convenience. The code couldn't handle all cases. Users always can manually + # download protoc from Protobuf's Github release page and pass the local path to the ONNX_CUSTOM_PROTOC_EXECUTABLE + # variable. + message("CMAKE_HOST_SYSTEM_NAME: ${CMAKE_HOST_SYSTEM_NAME}") + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") + if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "AMD64") + FetchContent_Declare(protoc_binary URL ${DEP_URL_protoc_win64} URL_HASH SHA1=${DEP_SHA1_protoc_win64}) + FetchContent_Populate(protoc_binary) + elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86") + FetchContent_Declare(protoc_binary URL ${DEP_URL_protoc_win32} URL_HASH SHA1=${DEP_SHA1_protoc_win32}) + FetchContent_Populate(protoc_binary) + endif() + if(protoc_binary_SOURCE_DIR) + message("Use prebuilt protoc") + set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${protoc_binary_SOURCE_DIR}/bin/protoc.exe) + set(PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE}) + endif() + elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64)$") + FetchContent_Declare(protoc_binary URL ${DEP_URL_protoc_linux_x64} URL_HASH SHA1=${DEP_SHA1_protoc_linux_x64}) + FetchContent_Populate(protoc_binary) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86?)$") + FetchContent_Declare(protoc_binary URL ${DEP_URL_protoc_linux_x86} URL_HASH SHA1=${DEP_SHA1_protoc_linux_x86}) + FetchContent_Populate(protoc_binary) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64.*") + FetchContent_Declare(protoc_binary URL ${DEP_URL_protoc_linux_aarch64} URL_HASH SHA1=${DEP_SHA1_protoc_linux_aarch64}) + FetchContent_Populate(protoc_binary) + endif() + if(protoc_binary_SOURCE_DIR) + message("Use prebuilt protoc") + set(ONNX_CUSTOM_PROTOC_EXECUTABLE ${protoc_binary_SOURCE_DIR}/bin/protoc) + set(PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE}) + endif() + endif() +endif() + #Here we support two build mode: #1. if ONNX_CUSTOM_PROTOC_EXECUTABLE is set, build Protobuf from source, except protoc.exe. This mode is mainly # for cross-compiling diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 523e7fb944..1065f7397e 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -2144,69 +2144,6 @@ def is_cross_compiling_on_apple(args): return False -# RID is short for runtime identifier. If a nuget package has native binaries, -# the RID designates on which platforms the package can be restored. However, Google's -# protobuf package doesn't use standard RIDs from .NET RID catalog. This function is -# specific for "google.protobuf.tools" nuget package -# We do not care which CPU arch this ONNX Runtime build is targeting, we only care -# the "host" CPU type. -def get_protobuf_rid(): - cpu_arch = platform.architecture()[0] - if is_windows(): - if platform.machine() == "AMD64": - # Even if cpu_arch is "32bit", we still use a 64-bit protoc binary because the CPU can run it - return "windows_x64" - # No ARM32/ARM64 support yet - # If you ran a x64 python exe on a Windows ARM64 machine, it will fall into the "windows_x64" branch above. - # If you ran native ARM64 python exe, we use "windows_x64" protoc.exe instead. - if platform.machine() == "ARM64": - return "windows_x64" - return None - if is_linux(): - # TODO: exclude ARM - if cpu_arch == "64bit": - return "linux_x64" - if cpu_arch == "32bit": - return "linux_x86" - return None - if is_macOS(): - # TODO: exclude ARM - return "macosx_x64" - return None - - -def build_protoc_for_host(cmake_path, source_dir, build_dir, args): - if (args.arm or args.arm64 or args.arm64ec) and not (is_windows() or is_cross_compiling_on_apple(args)): - raise BuildError( - "Currently only support building protoc for Windows host while " - "cross-compiling for ARM/ARM64/Store and linux cross-compiling iOS" - ) - - rid = get_protobuf_rid() - if rid is None: - return None - run_subprocess( - [ - "nuget.exe" if is_windows() else "nuget", - "restore", - os.path.join(source_dir, "packages.config"), - "-ConfigFile", - os.path.join(source_dir, "NuGet.config"), - "-PackagesDirectory", - build_dir, - ] - ) - - protoc_path = list(Path(build_dir).glob("Google.Protobuf.Tools.*"))[0] / "tools" / rid - if is_windows(): - protoc_path = protoc_path / "protoc.exe" - else: - protoc_path = protoc_path / "protoc" - if not protoc_path.exists(): - return None - return protoc_path.absolute() - - def generate_documentation(source_dir, build_dir, configs, validate): # Randomly choose one build config config = next(iter(configs)) @@ -2445,10 +2382,6 @@ def main(): ) cmake_extra_args = ["-G", args.cmake_generator] elif args.arm or args.arm64 or args.arm64ec: - # Cross-compiling for ARM(64) architecture - # First build protoc for host to use during cross-compilation - if path_to_protoc_exe is None: - path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args) if args.arm: cmake_extra_args = ["-A", "ARM"] elif args.arm64: @@ -2507,12 +2440,6 @@ def main(): log.info("Activating emsdk...") run_subprocess([emsdk_file, "activate", emsdk_version], cwd=emsdk_dir) - if ( - args.android or args.ios or args.build_wasm or is_cross_compiling_on_apple(args) - ) and args.path_to_protoc_exe is None: - # Cross-compiling for Android, iOS, and WebAssembly - path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args) - if is_ubuntu_1604(): if args.arm or args.arm64: raise BuildError("Only Windows ARM(64) cross-compiled builds supported currently through this script") diff --git a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml index d14a710632..585b88ed73 100644 --- a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml @@ -11,7 +11,7 @@ steps: packageType: upack feed: '/7424c8e4-5c62-490e-95c4-79446f31017c' definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0' - version: 1.0.46 + version: 1.0.52 downloadPath: $(Build.BinariesDirectory)/deps # The private ADO project @@ -22,7 +22,7 @@ steps: packageType: upack feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325' definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a' - version: 1.0.46 + version: 1.0.52 downloadPath: $(Build.BinariesDirectory)/deps # You can add more ADO accounts at here.