Added aarch64 build pipeline (#3841)

* Added aarch64 build pipeline

* Fix build error

* Remove auditwheel repair which doesn't work with cross compiling

* Statically link C++

* Added auditwheel repair back and fix stdlib.h

* Remove extra space

cmake/CMakeLists.txt

@@ -53,6 +53,7 @@ option(onnxruntime_USE_OPENBLAS "Use openblas" OFF)
 option(onnxruntime_DEV_MODE "Enable developer warnings and treat most of them as error." OFF)
 option(onnxruntime_USE_JEMALLOC "Use jemalloc" OFF)
 option(onnxruntime_MSVC_STATIC_RUNTIME "Compile for the static CRT" OFF)
+option(onnxruntime_GCC_STATIC_CPP_RUNTIME "Compile for the static libstdc++" OFF)
 option(onnxruntime_BUILD_UNIT_TESTS "Build ONNXRuntime unit tests" ON)
 option(onnxruntime_BUILD_CSHARP "Build C# library" OFF)
 option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF)
@@ -271,6 +272,9 @@ else()
     string(APPEND CMAKE_CXX_FLAGS " -mavx512f -mavx512cd -mavx512bw -mavx512dq -mavx512vl")
     string(APPEND CMAKE_C_FLAGS " -mavx512f -mavx512cd -mavx512bw -mavx512dq -mavx512vl")
   endif()
+  if(onnxruntime_GCC_STATIC_CPP_RUNTIME)
+    string(APPEND CMAKE_CXX_FLAGS " -static-libstdc++")
+  endif()
 endif()
 
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")

setup.py

@@ -67,10 +67,31 @@ elif '--use_nuphar' in sys.argv:
     sys.argv.remove('--use_nuphar')
 # --use_acl is specified in build.py, but not parsed here
 
-
-is_manylinux1 = False
-if environ.get('AUDITWHEEL_PLAT', None) == 'manylinux1_x86_64' or environ.get('AUDITWHEEL_PLAT', None) == 'manylinux2010_x86_64' :
-    is_manylinux1 = True
+# PEP 513 defined manylinux1_x86_64 and manylinux1_i686
+# PEP 571 defined manylinux2010_x86_64 and manylinux2010_i686
+# PEP 599 defines the following platform tags:
+# manylinux2014_x86_64
+# manylinux2014_i686
+# manylinux2014_aarch64
+# manylinux2014_armv7l
+# manylinux2014_ppc64
+# manylinux2014_ppc64le
+# manylinux2014_s390x
+manylinux_tags = [
+    'manylinux1_x86_64',
+    'manylinux1_i686',
+    'manylinux2010_x86_64',
+    'manylinux2010_i686',
+    'manylinux2014_x86_64',
+    'manylinux2014_i686',
+    'manylinux2014_aarch64',
+    'manylinux2014_armv7l',
+    'manylinux2014_ppc64',
+    'manylinux2014_ppc64le',
+    'manylinux2014_s390x',
+]
+ENV_AUDITWHEEL_PLAT = environ.get('AUDITWHEEL_PLAT', None)
+is_manylinux1 = ENV_AUDITWHEEL_PLAT in manylinux_tags
 
 
 class build_ext(_build_ext):

tools/ci_build/github/azure-pipelines/linux-arm-ci-pipeline.yml

@@ -0,0 +1,59 @@
+jobs:
+ - job: Linux_ARM
+   timeoutInMinutes: 60
+   pool: 'Linux-CPU'
+   strategy:
+     matrix:
+        Py37:
+            python.include: '3.7m'
+            cp.tag: 'cp37-cp37m'
+        Py36:
+            python.include: '3.6m'
+            cp.tag: 'cp36-cp36m'
+        Py35:
+            python.include: '3.5m'
+            cp.tag: 'cp35-cp35m'
+   steps:
+     - task: CmdLine@2
+       inputs:
+         script: |
+           set -e -x
+           sudo rm -rf *
+           cd $(Build.SourcesDirectory)
+           git submodule update --init --recursive
+           cd -
+           sudo apt-get install -y qemu-user-static
+           sudo chmod a+x /usr/bin/azcopy
+
+           cat << EOF > tool-chain.cmake
+           SET(CMAKE_SYSTEM_NAME Linux)
+           SET(CMAKE_SYSTEM_VERSION 1)
+           SET(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
+           set(CMAKE_C_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits")
+           SET(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)
+           set(CMAKE_CXX_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits")
+           SET(CMAKE_FIND_ROOT_PATH /mnt/toolchains/manylinux2014_aarch64)
+           SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+           SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+           SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+           SET(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+           EOF
+           export PATH=/mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/bin:$PATH
+           azcopy cp https://onnxruntimetestdata.blob.core.windows.net/models/toolchains.tar.xz $(Build.BinariesDirectory)/toolchains.tar.xz
+           sudo rm -rf /mnt/toolchains
+           mkdir /mnt/toolchains
+           tar -Jxf $(Build.BinariesDirectory)/toolchains.tar.xz -C /mnt/toolchains
+           aria2c -q https://github.com/protocolbuffers/protobuf/releases/download/v3.11.1/protoc-3.11.1-linux-x86_64.zip
+           unzip protoc-3.11.1-linux-x86_64.zip
+           aria2c -q https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz
+           tar --strip=1 -zxf cmake-3.17.1-Linux-x86_64.tar.gz
+           sudo cp /mnt/toolchains/manylinux2014_aarch64/usr/include/stdlib.h /mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/aarch64-linux-gnu/libc/usr/include/
+           bin/cmake -Donnxruntime_GCC_STATIC_CPP_RUNTIME=ON -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB=OFF -DCMAKE_TOOLCHAIN_FILE=tool-chain.cmake -Donnxruntime_ENABLE_PYTHON=ON -DPYTHON_LIBRARY=dl -DPYTHON_EXECUTABLE=/mnt/toolchains/manylinux2014_aarch64/opt/python/'$(cp.tag)'/bin/python3  -Donnxruntime_BUILD_SHARED_LIB=OFF  -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_DEV_MODE=ON -DONNX_CUSTOM_PROTOC_EXECUTABLE=$(Build.BinariesDirectory)/bin/protoc "-DPYTHON_INCLUDE_DIR=/mnt/toolchains/manylinux2014_aarch64/usr/include;/mnt/toolchains/manylinux2014_aarch64/opt/python/$(cp.tag)/include/python$(python.include)" -DNUMPY_INCLUDE_DIR=/mnt/toolchains $(Build.SourcesDirectory)/cmake
+           make -j$(getconf _NPROCESSORS_ONLN)
+           docker run -v /usr/bin/qemu-aarch64-static:/usr/bin/qemu-aarch64-static -v $(Build.BinariesDirectory):/tmp/a -v $(Build.SourcesDirectory):/tmp/b -w /tmp/a --rm quay.io/pypa/manylinux2014_aarch64 /opt/python/'$(cp.tag)'/bin/python3 /tmp/b/setup.py bdist_wheel
+         workingDirectory: $(Build.BinariesDirectory)
+     - task: PublishBuildArtifacts@1
+       displayName: 'Publish Artifact: wheels'
+       inputs:
+         PathtoPublish: '$(Build.BinariesDirectory)/dist'
+         ArtifactName: wheels