Support Windows cross-compiling for ARM(64) in ORT build scripts (#549)

* Initial commit * More changes * More changes * More changes * More changes * PR feedback * Commiting Azure build config file * Fix build pipeline * Cleanup build dir template addition * Remove conda modules download step * PR feedback * Revert x86 arguments to as they are currently * More changes
2026-06-29 03:30:52 +00:00 · 2019-03-08 17:42:20 -08:00 · 2019-03-08 17:42:20 -08:00 · 867eda5262
commit 867eda5262
parent 6136efc0c0
5 changed files with 181 additions and 10 deletions
--- a/BUILD.md
+++ b/BUILD.md
@ -52,7 +52,7 @@ ONNX Runtime python binding only supports Python 3.x. Please use python 3.5+.
    ```
 5. Run `./build.sh --config RelWithDebInfo --build_wheel` for Linux (or `build.bat --config RelWithDebInfo --build_wheel` for Windows)

-The build script runs all unit tests by default.
+The build script runs all unit tests by default (for native builds and skips tests by default for cross-compiled builds).

 The complete list of build options can be found by running `./build.sh (or ./build.bat) --help`

@ -197,4 +197,11 @@ Please see [ARM docker file](dockerfiles/Dockerfile.arm32v7). Docker build runs
 By doing this, you could avoid hit the ACR-Tasks build timeout (8 hours) 

 ### Cross compiling on Windows
-(TODO)
+#### Using Visual C++ compilers
+1. Download and install Visual C++ compilers and libraries for ARM(64). 
+   If you have Visual Studio installed, please use the Visual Studio Installer (look under the section `Individual components` after choosing to `modify` Visual Studio) to download and install the corresponding ARM(64) compilers and libraries.
+   
+2. Use `build.bat` and specify `--arm` or `--arm64` as the build option to start building. Preferably use `Developer Command Prompt for VS` or make sure all the installed cross-compilers are findable from the command prompt being used to build using the PATH environmant variable. 
+
+### Using other compilers
+(TODO)
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -65,6 +65,7 @@ option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable
 option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF)
 option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF)
 option(onnxruntime_USE_TRT "Build with TensorRT support" OFF)
+option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)

 set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
 #nsync tests failed on Mac Build
@ -84,6 +85,10 @@ if(onnxruntime_USE_OPENMP)
  endif()
 endif()

+if(onnxruntime_CROSS_COMPILING)
+  set(CMAKE_CROSSCOMPILING ON)  
+endif()
+
 #must after OpenMP settings
 find_package(Threads)

@ -340,8 +345,9 @@ if (WIN32)
        string(APPEND CMAKE_CXX_FLAGS " /W4")
    endif()

-    # treat warning as error only on x64 platform. For x86, there are too many warnings to fix.
-    if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE)
+    # treat warning as error only on x64 platform. 
+    # For x86 and cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now
+    if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING)
      # treat warnings as errors
      string(APPEND CMAKE_CXX_FLAGS " /WX")
      foreach(type EXE STATIC SHARED)
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -38,7 +38,8 @@ class UsageError(BaseError):
 def parse_arguments():
    parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.",
                                     usage='''
-Default behavior is --update --build --test.
+Default behavior is --update --build --test for native architecture builds.
+Default behavior is --update --build for cross-compiled builds.

 The Update phase will update git submodules, and run cmake to generate makefiles.
 The Build phase will build all projects.
@ -64,6 +65,7 @@ Use the individual flags to only run the specified stages.
    parser.add_argument("--enable_onnx_tests", action='store_true',
                        help='''When running the Test phase, run onnx_test_running against available test data directories.''')
    parser.add_argument("--pb_home", help="Path to protobuf installation")
+    parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. Will be overridden by {pb_home}/bin/protoc.exe if {pb_home} is set.")
    parser.add_argument("--download_test_data", action="store_true",
                        help='''Downloads test data without running the tests''')
    parser.add_argument("--test_data_url", help="Test data URL.")
@ -93,6 +95,10 @@ Use the individual flags to only run the specified stages.
                             "These are just CMake -D options without the leading -D.")
    parser.add_argument("--x86", action='store_true',
                        help="Create x86 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
+    parser.add_argument("--arm", action='store_true',
+                        help="Create ARM makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
+    parser.add_argument("--arm64", action='store_true',
+                        help="Create ARM64 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed")
    parser.add_argument("--msvc_toolset", help="MSVC toolset to use. e.g. 14.11")

    # Arguments needed by CI
@ -271,7 +277,7 @@ def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure
                log.debug("creating shortcut %s -> %s"  % (src_model_dir, dest_model_dir))
                run_subprocess(['mklink', '/D', '/J', dest_model_dir, src_model_dir], shell=True)

-def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, configs, cmake_extra_defines, args, cmake_extra_args):
+def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
    log.info("Generating CMake build tree")
    cmake_dir = os.path.join(source_dir, "cmake")
    # TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind)
@ -300,6 +306,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                 "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"),
                 "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), 
                 "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"),
+                  # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script)
+                 "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"),
                 ]
    if args.use_brainslice:
        bs_pkg_name = args.brain_slice_package_name.split('.', 1)
@ -326,6 +334,9 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
    if pb_home:
        cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON']

+    elif path_to_protoc_exe:
+        cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % path_to_protoc_exe]
+
    cmake_args += ["-D{}".format(define) for define in cmake_extra_defines]

    if is_windows():
@ -518,17 +529,48 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda):
        if is_ubuntu_1604():
            run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist')

+def build_protoc_for_windows_host(cmake_path, source_dir, build_dir):
+    if not is_windows():
+        raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM/ARM64 arch')
+
+    log.info("Building protoc for host to be used in cross-compiled build process")
+    protoc_build_dir = os.path.join(build_dir, 'host_protoc')
+    os.makedirs(protoc_build_dir, exist_ok=True)
+    # Generate step
+    cmd_args = [cmake_path,
+                os.path.join(source_dir, 'cmake\external\protobuf\cmake'),
+                '-T',
+                'host=x64',
+                '-G',
+                'Visual Studio 15 2017',
+                '-Dprotobuf_BUILD_TESTS=OFF',
+                '-Dprotobuf_WITH_ZLIB_DEFAULT=OFF',
+                '-Dprotobuf_BUILD_SHARED_LIBS=OFF']
+    run_subprocess(cmd_args, cwd= protoc_build_dir)
+    # Build step
+    cmd_args = [cmake_path,
+                "--build", protoc_build_dir,
+                "--config", "Release",
+                "--target", "protoc"]
+    run_subprocess(cmd_args)
+
+    if not os.path.exists(os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')):
+        raise BuildError("Couldn't build protoc.exe for host. Failing build.")
+
 def main():
    args = parse_arguments()

    cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else []

-    # if there was no explicit argument saying what to do, default to update, build and test.
+    # if there was no explicit argument saying what to do, default to update, build and test (for native builds).
    if (args.update == False and args.clean == False and args.build == False and args.test == False):
-        log.debug("Defaulting to running update, build and test.")
+        log.debug("Defaulting to running update, build [and test for native builds].")
        args.update = True
        args.build = True
-        args.test = True
+        if args.arm or args.arm64:
+            args.test = False
+        else:
+            args.test = True

    if args.build_wheel:
        args.enable_pybind = True
@ -557,6 +599,19 @@ def main():
        if(is_windows()):
          if (args.x86):
            cmake_extra_args = ['-A','Win32','-T','host=x64','-G', 'Visual Studio 15 2017']
+          elif (args.arm or args.arm64):
+            # Cross-compiling for ARM(64) architecture
+            # First build protoc for host to use during cross-compilation
+            build_protoc_for_windows_host(cmake_path, source_dir, build_dir)
+            if args.arm:
+                cmake_extra_args = ['-A', 'ARM']
+            else:
+                cmake_extra_args = ['-A', 'ARM64']
+            cmake_extra_args += ['-G', 'Visual Studio 15 2017']
+            # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
+            if args.test:
+                log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
+                args.test = False
          else:
            toolset = 'host=x64'
            if (args.msvc_toolset):
@ -566,6 +621,8 @@ def main():

            cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017']
        if is_ubuntu_1604():
+            if (args.arm or args.arm64):
+                raise BuildError("Only Windows ARM(64) cross-compiled builds supported currently through this script")
            install_ubuntu_deps(args)
            if not is_docker():
                install_python_deps()
@ -579,7 +636,14 @@ def main():
               raise UsageError("The test_data_url and test_data_checksum arguments are required.")
            setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key)

-        generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, configs, cmake_extra_defines,
+        path_to_protoc_exe = None
+        if args.path_to_protoc_exe:
+            path_to_protoc_exe = args.path_to_protoc_exe
+        # Need to provide path to protoc.exe built for host to be used in the cross-compiled build process
+        elif args.arm or args.arm64:
+            path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')
+
+        generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines,
                            args, cmake_extra_args)

    if (args.clean):
--- a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml
@ -0,0 +1,47 @@
+jobs:
+- job: Windows_ARM_CrossCompile_CI_Dev
+  variables:
+    buildDirectory: '$(Build.BinariesDirectory)'
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+    - task: UniversalPackages@0
+      displayName: 'Download python'
+      inputs:
+        command: download
+        vstsFeed: '$(System.TeamProject)'
+        vstsFeedPackage: 'miniconda3_win64'
+        vstsPackageVersion: '4.5.11'
+        downloadDirectory: '$(Build.BinariesDirectory)\python'
+    - task: CmdLine@1
+      displayName: 'Run python installer'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\python\installer.exe'
+        arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
+      timeoutInMinutes: 10
+    - task: BatchScript@1
+      displayName: 'setup env'
+      inputs:
+        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Download cmake'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Debug'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Release'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+      displayName: 'Component Detection'
+      condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))   
+    - template: templates/clean-agent-build-directory-step.yml
--- a/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml
@ -0,0 +1,47 @@
+jobs:
+- job: Windows_ARM_CrossCompile_CI_Dev
+  variables:
+    buildDirectory: '$(Build.BinariesDirectory)'
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+    - task: UniversalPackages@0
+      displayName: 'Download python'
+      inputs:
+        command: download
+        vstsFeed: '$(System.TeamProject)'
+        vstsFeedPackage: 'miniconda3_win64'
+        vstsPackageVersion: '4.5.11'
+        downloadDirectory: '$(Build.BinariesDirectory)\python'
+    - task: CmdLine@1
+      displayName: 'Run python installer'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\python\installer.exe'
+        arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
+      timeoutInMinutes: 10
+    - task: BatchScript@1
+      displayName: 'setup env'
+      inputs:
+        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Download cmake'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Debug'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - task: CmdLine@1
+      displayName: 'Generate cmake config and build Release'
+      inputs:
+        filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+        arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64'
+        workingDirectory: "$(Build.BinariesDirectory)"
+    - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+      displayName: 'Component Detection'
+      condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))        
+    - template: templates/clean-agent-build-directory-step.yml