Update C/C++ dependencies: abseil, date, nsync, googletest, wil, mp11, cpuinfo and safeint (#15470)

### Description
Update C/C++ dependencies abseil, date, nsync, googletest, wil, mp11,
cpuinfo and safeint to newer versions per request of @
mayeut. He created the following PRs to update the deps:
https://github.com/microsoft/onnxruntime/pull/15432
https://github.com/microsoft/onnxruntime/pull/15434
https://github.com/microsoft/onnxruntime/pull/15435
https://github.com/microsoft/onnxruntime/pull/15436
https://github.com/microsoft/onnxruntime/pull/15437

However, our build system needs to fetch the dependencies from an
internal mirror that only Microsoft employees have write access to. So I
closed his PRs and created this one.

This PR also updates abseil to a newer version. This is to prepare for
upgrading re2.
This commit is contained in:
Changming Sun 2023-09-08 13:35:04 -07:00 committed by GitHub
parent f51a765e64
commit bc84f52633
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 156 additions and 252 deletions

View file

@ -91,7 +91,7 @@ def add_github_dep(name, parsed_url):
with open(
os.path.join(REPO_DIR, "tools", "ci_build", "github", "linux", "docker", "Dockerfile.manylinux2014_cuda11"),
os.path.join(REPO_DIR, "tools", "ci_build", "github", "linux", "docker", "Dockerfile.manylinux2_28_cuda11"),
) as f:
for line in f:
if not line.strip():

View file

@ -112,7 +112,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "8c0b94e793a66495e0b1f34a5eb26bd7dc672db0",
"commitHash": "29bf8085f3bf17b84d30e34b3d7ff8248fda404e",
"repositoryUrl": "https://github.com/abseil/abseil-cpp.git"
},
"comments": "abseil_cpp"
@ -132,7 +132,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "e7e1482087f58913b80a20b04d5c58d9d6d90155",
"commitHash": "6e921e1b1d21e84a5c82416ba7ecd98e33a436d0",
"repositoryUrl": "https://github.com/HowardHinnant/date.git"
},
"comments": "date"
@ -192,7 +192,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "436617053d0f39a1019a371c3a9aa599b3cb2cea",
"commitHash": "13de152c2a1cd73ff4df97bd2c406b6d15d34af3",
"repositoryUrl": "https://github.com/google/nsync.git"
},
"comments": "google_nsync"
@ -202,7 +202,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "519beb0e52c842729b4b53731d27c0e0c32ab4a2",
"commitHash": "f8d7d77c06936315286eb55f8de22cd23c188571",
"repositoryUrl": "https://github.com/google/googletest.git"
},
"comments": "googletest"
@ -242,7 +242,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "5f4caba4e7a9017816e47becdd918fcc872039ba",
"commitHash": "55f373e164d3f092dd6c7a56e3de6f90c4c6f3dc",
"repositoryUrl": "https://github.com/microsoft/wil.git"
},
"comments": "microsoft_wil"
@ -262,7 +262,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "7bc4e1ae9b36ec8ee635c3629b59ec525bbe82b9",
"commitHash": "0a0b5fb001ce0233ae3a6f99d849c0649e5a7361",
"repositoryUrl": "https://github.com/boostorg/mp11.git"
},
"comments": "mp11"
@ -322,7 +322,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "5916273f79a21551890fd3d56fc5375a78d1598d",
"commitHash": "959002f82d7962a473d8bf301845f2af720e0aa4",
"repositoryUrl": "https://github.com/pytorch/cpuinfo.git"
},
"comments": "pytorch_cpuinfo"
@ -342,7 +342,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "ff15c6ada150a5018c5ef2172401cb4529eac9c0",
"commitHash": "4cafc9196c4da9c817992b20f5253ef967685bf8",
"repositoryUrl": "https://github.com/dcleblanc/SafeInt.git"
},
"comments": "safeint"
@ -368,6 +368,16 @@
"comments": "cutlass"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "72c943dea2b9240cd09efde15191e144bc7c7d38",
"repositoryUrl": "https://github.com/protocolbuffers/utf8_range.git"
},
"comments": "utf8_range"
}
},
{
"component": {
"type": "git",

View file

@ -554,6 +554,8 @@ if (WIN32)
list(APPEND ORT_WARNING_FLAGS "/wd4251")
# issued by thrust nonstandard extension used: nameless struct/union
list(APPEND ORT_WARNING_FLAGS "/wd4201")
# structure was padded due to __declspec(align())
list(APPEND ORT_WARNING_FLAGS "/wd4324")
# warning C4800: Implicit conversion from 'X' to bool. Possible information loss
if (onnxruntime_USE_OPENVINO)
list(APPEND ORT_WARNING_FLAGS "/wd4800")

View file

@ -5,24 +5,25 @@
#URLs can be either https URLs or local file paths in cmake-style(directory separator is a forward slash character).
#SHA1 hashes can be generated by running sha1sum command.
#If you need to change abseil's version to a different one, you may also want to update external\abseil-cpp.natvis
#since the file contains a version string: "lts_20220623". However, the file is for debugging purposes only and would
#since the file contains a version string: "lts_20230802". However, the file is for debugging purposes only and would
#not affect built binaries.
abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.zip;50c137c88965cba015dfcc8fd5d9b46d23146751
abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.zip;04271dfbfac59269b6939e1e9d5faf0d18a7ba91
cxxopts;https://github.com/jarro2783/cxxopts/archive/3c73d91c0b04e2b59462f0a741be8c07024c1bc0.zip;6c6ca7f8480b26c8d00476e0e24b7184717fe4f0
date;https://github.com/HowardHinnant/date/archive/refs/tags/v2.4.1.zip;ea99f021262b1d804a872735c658860a6a13cc98
date;https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.zip;2dac0c81dc54ebdd8f8d073a75c053b04b56e159
dlpack;https://github.com/dmlc/dlpack/archive/refs/tags/v0.6.zip;4d565dd2e5b31321e5549591d78aa7f377173445
eigen;https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.zip;ee201b07085203ea7bd8eb97cbcb31b07cfa3efb
flatbuffers;https://github.com/google/flatbuffers/archive/refs/tags/v1.12.0.zip;ba0a75fd12dbef8f6557a74e611b7a3d0c5fe7bf
fp16;https://github.com/Maratyszcza/FP16/archive/0a92994d729ff76a58f692d3028ca1b64b145d91.zip;b985f6985a05a1c03ff1bb71190f66d8f98a1494
fxdiv;https://github.com/Maratyszcza/FXdiv/archive/63058eff77e11aa15bf531df5dd34395ec3017c8.zip;a5658f4036402dbca7cebee32be57fb8149811e1
google_benchmark;https://github.com/google/benchmark/archive/refs/tags/v1.7.0.zip;e97c368b176e8614e3f1bf13dd9abcf6a7ad9908
google_nsync;https://github.com/google/nsync/archive/refs/tags/1.23.0.zip;f3233450cf7156fc0bedd1b0e884eddec264897c
googletest;https://github.com/google/googletest/archive/519beb0e52c842729b4b53731d27c0e0c32ab4a2.zip;4b3c37972e4c1bef1185d46f702082f8772ee73f
google_nsync;https://github.com/google/nsync/archive/refs/tags/1.26.0.zip;5e7c00ef6bf5b787386fc040067903ec774e2752
googletest;https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip;0ac421f2ec11af38b0fff0f1992184032731a8bc
googlexnnpack;https://github.com/google/XNNPACK/archive/003c580e696a774afdc984996ee909b7c8d8128c.zip;9f192e3f15e1e37ae9c78d53eeea47e45c5eb31c
json;https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.zip;f257f8dc27c5b8c085dc887b40cddd18ae1f725c
microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf368104cd22a87b4dd0c80228919bb2df3e2a14
microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becdd918fcc872039ba.zip;fd119887d0d17c37adf1fc227b054befa28158ad
microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5
mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41
mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5
mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.82.0.zip;9bc9e01dffb64d9e0773b2e44d2f22c51aace063
onnx;https://github.com/onnx/onnx/archive/e2525550194ce3d8a2c4a3af451c9d9b3ae6650e.zip;782f23d788185887f520a90535513e244218e928
#use the commit of supporting all the plugins and TRT 8.6-GA (https://github.com/onnx/onnx-tensorrt/commit/0462dc31ae78f48744b6141ae376df1f96d3f459)
onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/0462dc31ae78f48744b6141ae376df1f96d3f459.zip;5ff086361956cceb81ed17453a1fd8db2aa4328d
@ -36,11 +37,11 @@ protoc_mac_universal;https://github.com/protocolbuffers/protobuf/releases/downlo
psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013
pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101
pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13
pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/5916273f79a21551890fd3d56fc5375a78d1598d.zip;2be4d2ae321fada97cb39eaf4eeba5f8c85597cf
pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/959002f82d7962a473d8bf301845f2af720e0aa4.zip;85da3caa60eb2b148613b443fbc2bfdc30689965
re2;https://github.com/google/re2/archive/refs/tags/2022-06-01.zip;aa77313b76e91b531ee7f3e45f004c6a502a5374
safeint;https://github.com/dcleblanc/SafeInt/archive/ff15c6ada150a5018c5ef2172401cb4529eac9c0.zip;913a4046e5274d329af2806cb53194f617d8c0ab
safeint;https://github.com/dcleblanc/SafeInt/archive/refs/tags/3.0.28.zip;23f252040ff6cb9f1fd18575b32fa8fb5928daac
tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81.zip;67b833913605a4f3f499894ab11528a702c2b381
cutlass;https://github.com/NVIDIA/cutlass/archive/refs/tags/v3.0.0.zip;0f95b3c1fc1bd1175c4a90b2c9e39074d1bccefd
utf8_range;https://github.com/protocolbuffers/utf8_range/archive/72c943dea2b9240cd09efde15191e144bc7c7d38.zip;9925739c9debc0efa2adcb194d371a35b6a03156
extensions;https://github.com/microsoft/onnxruntime-extensions/archive/94142d8391c9791ec71c38336436319a2d4ac7a0.zip;4365ac5140338b4cb75a39944a4be276e3829b3c
eigen;https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.zip;ee201b07085203ea7bd8eb97cbcb31b07cfa3efb
composable_kernel;https://github.com/ROCmSoftwarePlatform/composable_kernel/archive/d52ec01652b7d620386251db92455968d8d90bdc.zip;6b5ce8edf3625f8817086c194fbf94b664e1b0e0
composable_kernel;https://github.com/ROCmSoftwarePlatform/composable_kernel/archive/d52ec01652b7d620386251db92455968d8d90bdc.zip;6b5ce8edf3625f8817086c194fbf94b664e1b0e0

View file

@ -6,15 +6,20 @@ include(FetchContent)
# Pass to build
set(ABSL_PROPAGATE_CXX_STD 1)
set(BUILD_TESTING 0)
set(ABSL_BUILD_TESTING OFF)
set(ABSL_BUILD_TEST_HELPERS OFF)
set(ABSL_USE_EXTERNAL_GOOGLETEST ON)
if(Patch_FOUND AND WIN32)
set(ABSL_PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/abseil/absl_windows.patch)
else()
set(ABSL_PATCH_COMMAND "")
endif()
if(WIN32 AND NOT Patch_FOUND)
#see https://github.com/google/re2/issues/425 and https://github.com/google/re2/issues/436
set(ABSL_ENABLE_INSTALL ON)
endif()
# NB! Advancing Abseil version changes its internal namespace,
# currently absl::lts_20211102 which affects abseil-cpp.natvis debugger
# currently absl::lts_20230125 which affects abseil-cpp.natvis debugger
# visualization file, that must be adjusted accordingly, unless we eliminate
# that namespace at build time.
FetchContent_Declare(
@ -22,6 +27,7 @@ FetchContent_Declare(
URL ${DEP_URL_abseil_cpp}
URL_HASH SHA1=${DEP_SHA1_abseil_cpp}
PATCH_COMMAND ${ABSL_PATCH_COMMAND}
FIND_PACKAGE_ARGS NAMES absl
)
onnxruntime_fetchcontent_makeavailable(abseil_cpp)
@ -37,8 +43,26 @@ if (GDK_PLATFORM)
target_compile_definitions(absl_symbolize PRIVATE WINAPI_FAMILY=WINAPI_FAMILY_DESKTOP_APP)
endif()
if(NOT onnxruntime_DISABLE_ABSEIL)
set(ABSEIL_LIBS absl::inlined_vector absl::flat_hash_set
absl::flat_hash_map absl::node_hash_set absl::node_hash_map absl::base absl::throw_delegate absl::raw_hash_set
absl::hash absl::city absl::low_level_hash absl::raw_logging_internal)
endif()
# TODO: since multiple ORT's dependencies depend on Abseil, the list below would vary from version to version.
# We'd better to not manually manage the list.
set(ABSEIL_LIBS absl::base
absl::city
absl::core_headers
absl::fixed_array
absl::flags
absl::flat_hash_map
absl::flat_hash_set
absl::hash
absl::inlined_vector
absl::low_level_hash
absl::node_hash_map
absl::node_hash_set
absl::optional
absl::raw_hash_set
absl::raw_logging_internal
absl::span
absl::str_format
absl::strings
absl::synchronization
absl::throw_delegate
absl::time)

View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
<Type Name="absl::lts_20220623::InlinedVector&lt;*&gt;">
<Type Name="absl::lts_20230802::InlinedVector&lt;*&gt;">
<Intrinsic Name="_size" Expression="storage_.metadata_.value >> 1"/>
<Intrinsic Name="_is_allocated" Expression="(storage_.metadata_.value &amp; 1) == 1"/>
<Intrinsic Name="_inlined_data" Expression="($T1*)storage_.data_.inlined.inlined_data"/>
@ -24,7 +24,7 @@
</Expand>
</Type>
<!-- Should handle both flat hash_set and hash_map -->
<Type Name="absl::lts_20220623::container_internal::raw_hash_set&lt;*&gt;">
<Type Name="absl::lts_20230802::container_internal::raw_hash_set&lt;*&gt;">
<DisplayString Condition="size_ == 0">empty</DisplayString>
<DisplayString>{{ size={size_} }}</DisplayString>
<Expand>
@ -44,7 +44,7 @@
</CustomListItems>
</Expand>
</Type>
<Type Name="absl::lts_20220623::container_internal::map_slot_type&lt;*&gt;">
<Type Name="absl::lts_20230802::container_internal::map_slot_type&lt;*&gt;">
<DisplayString>{{ {value.first}:{value.second} }}</DisplayString>
<Expand>
<Item Name="[key]" ExcludeView="simple">value.first</Item>

View file

@ -22,7 +22,8 @@ if (onnxruntime_REDUCED_OPS_BUILD)
endif()
if (onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS)
set(OCOS_ENABLE_SPM_TOKENIZER ON CACHE INTERNAL "")
#The generated protobuf files in ORT-extension needs be updated to work with the current protobuf version ORT is using.
set(OCOS_ENABLE_SPM_TOKENIZER OFF CACHE INTERNAL "")
set(OCOS_ENABLE_GPT2_TOKENIZER ON CACHE INTERNAL "")
set(OCOS_ENABLE_WORDPIECE_TOKENIZER ON CACHE INTERNAL "")
set(OCOS_ENABLE_BERT_TOKENIZER ON CACHE INTERNAL "")
@ -54,9 +55,11 @@ endif()
target_include_directories(ocos_operators PRIVATE ${RE2_INCLUDE_DIR} ${json_SOURCE_DIR}/include)
target_include_directories(ortcustomops PUBLIC ${onnxruntime_EXTENSIONS_PATH}/includes)
if(OCOS_ENABLE_SPM_TOKENIZER)
onnxruntime_add_include_to_target(sentencepiece-static ${PROTOBUF_LIB})
onnxruntime_add_include_to_target(sentencepiece-static ${PROTOBUF_LIB} ${ABSEIL_LIBS})
endif()
onnxruntime_add_include_to_target(ocos_operators ${PROTOBUF_LIB})
onnxruntime_add_include_to_target(ocos_operators ${PROTOBUF_LIB} ${ABSEIL_LIBS})
onnxruntime_add_include_to_target(noexcep_operators ${PROTOBUF_LIB} ${ABSEIL_LIBS})
add_dependencies(ocos_operators ${onnxruntime_EXTERNAL_DEPENDENCIES})
add_dependencies(ortcustomops ${onnxruntime_EXTERNAL_DEPENDENCIES})

View file

@ -19,11 +19,10 @@ endforeach()
message("Loading Dependencies ...")
# ABSL should be included before protobuf because protobuf may use absl
if(NOT onnxruntime_DISABLE_ABSEIL)
include(external/abseil-cpp.cmake)
endif()
include(external/abseil-cpp.cmake)
set(RE2_BUILD_TESTING OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
re2
URL ${DEP_URL_re2}
@ -38,10 +37,8 @@ if (onnxruntime_BUILD_UNIT_TESTS)
set(gtest_disable_pthreads ON)
endif()
set(INSTALL_GTEST OFF CACHE BOOL "" FORCE)
if(NOT onnxruntime_DISABLE_ABSEIL)
# It uses both ABSL and re2
set(GTEST_HAS_ABSL OFF CACHE BOOL "" FORCE)
endif()
# Set it to ON will cause crashes in onnxruntime_test_all when onnxruntime_USE_CUDA is ON
set(GTEST_HAS_ABSL OFF CACHE BOOL "" FORCE)
# gtest and gmock
FetchContent_Declare(
googletest
@ -161,6 +158,19 @@ if(Patch_FOUND)
else()
set(ONNXRUNTIME_PROTOBUF_PATCH_COMMAND "")
endif()
FetchContent_Declare(
utf8_range
URL ${DEP_URL_utf8_range}
URL_HASH SHA1=${DEP_SHA1_utf8_range}
FIND_PACKAGE_ARGS NAMES utf8_range
)
set(utf8_range_ENABLE_TESTS OFF CACHE BOOL "Build test suite" FORCE)
set(utf8_range_ENABLE_INSTALL OFF CACHE BOOL "Configure installation" FORCE)
#Protobuf depends on absl and utf8_range
FetchContent_Declare(
Protobuf
URL ${DEP_URL_protobuf}
@ -168,7 +178,15 @@ FetchContent_Declare(
PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND}
FIND_PACKAGE_ARGS 3.21.12 NAMES Protobuf
)
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
#TODO: we'd better to turn the following option off. However, it will cause
# ".\build.bat --config Debug --parallel --skip_submodule_sync --update" fail with an error message:
# install(EXPORT "ONNXTargets" ...) includes target "onnx_proto" which requires target "libprotobuf-lite" that is
# not in any export set.
#set(protobuf_INSTALL OFF CACHE BOOL "Install protobuf binaries and files" FORCE)
set(protobuf_USE_EXTERNAL_GTEST ON CACHE BOOL "" FORCE)
if (CMAKE_SYSTEM_NAME STREQUAL "Android")
set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build protobuf tests" FORCE)
set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build with zlib support" FORCE)
@ -184,14 +202,13 @@ set(ENABLE_DATE_TESTING OFF CACHE BOOL "" FORCE)
set(USE_SYSTEM_TZ_DB ON CACHE BOOL "" FORCE)
FetchContent_Declare(
date
URL ${DEP_URL_date}
URL_HASH SHA1=${DEP_SHA1_date}
)
date
URL ${DEP_URL_date}
URL_HASH SHA1=${DEP_SHA1_date}
FIND_PACKAGE_ARGS 3...<4 NAMES date
)
onnxruntime_fetchcontent_makeavailable(date)
FetchContent_Declare(
mp11
URL ${DEP_URL_mp11}
@ -252,6 +269,20 @@ else()
set(CPUINFO_SUPPORTED FALSE)
endif()
# xnnpack depends on clog
# Android build should use the system's log library instead of clog
if ((CPUINFO_SUPPORTED OR onnxruntime_USE_XNNPACK) AND NOT ANDROID)
set(CLOG_BUILD_TESTS OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
pytorch_clog
URL ${DEP_URL_pytorch_cpuinfo}
URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
SOURCE_SUBDIR deps/clog
)
set(ONNXRUNTIME_CLOG_PROJ pytorch_clog)
set(ONNXRUNTIME_CLOG_TARGET_NAME clog)
endif()
if (CPUINFO_SUPPORTED)
if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
set(IOS ON CACHE INTERNAL "")
@ -276,7 +307,7 @@ if (CPUINFO_SUPPORTED)
URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
FIND_PACKAGE_ARGS NAMES cpuinfo
)
set(ONNXRUNTIME_CPUINFO_PROJ pytorch_cpuinfo)
endif()
@ -316,8 +347,14 @@ FetchContent_Declare(
URL_HASH SHA1=${DEP_SHA1_safeint}
)
# use fetch content rather than makeavailable because safeint only includes unconditional test targets
FetchContent_Populate(safeint)
# The next line will generate an error message "fatal: not a git repository", but it is ok. It is from flatbuffers
onnxruntime_fetchcontent_makeavailable(Protobuf nlohmann_json mp11 re2 safeint GSL flatbuffers)
onnxruntime_fetchcontent_makeavailable(utf8_range)
# protobuf's cmake/utf8_range.cmake has the following line
include_directories(${utf8_range_SOURCE_DIR})
onnxruntime_fetchcontent_makeavailable(Protobuf nlohmann_json mp11 re2 GSL flatbuffers ${ONNXRUNTIME_CPUINFO_PROJ} ${ONNXRUNTIME_CLOG_PROJ})
if(NOT flatbuffers_FOUND)
if(NOT TARGET flatbuffers::flatbuffers)
add_library(flatbuffers::flatbuffers ALIAS flatbuffers)
@ -413,15 +450,7 @@ FetchContent_Declare(
)
if (CPUINFO_SUPPORTED)
onnxruntime_fetchcontent_makeavailable(pytorch_cpuinfo)
if (pytorch_cpuinfo_SOURCE_DIR)
# shouldn't need to define these aliases after we use a version of cpuinfo with this commit:
# https://github.com/pytorch/cpuinfo/commit/082deffc80ce517f81dc2f3aebe6ba671fcd09c9
add_library(cpuinfo::cpuinfo ALIAS cpuinfo)
add_library(cpuinfo::clog ALIAS clog)
endif()
endif()
@ -462,7 +491,7 @@ endif()
#onnxruntime_EXTERNAL_LIBRARIES could contain onnx, onnx_proto,libprotobuf, cuda/cudnn,
# dnnl/mklml, onnxruntime_codegen_tvm, tvm and pthread
# pthread is always at the last
set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} ${WIL_TARGET} nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface flatbuffers::flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS} date_interface)
set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} ${WIL_TARGET} nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface flatbuffers::flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS} date::date ${ONNXRUNTIME_CLOG_TARGET_NAME})
# The source code of onnx_proto is generated, we must build this lib first before starting to compile the other source code that uses ONNX protobuf types.
# The other libs do not have the problem. All the sources are already there. We can compile them in any order.
set(onnxruntime_EXTERNAL_DEPENDENCIES onnx_proto flatbuffers::flatbuffers)

View file

@ -121,7 +121,7 @@ if (MSVC)
endif()
endif()
onnxruntime_add_include_to_target(onnxruntime_common date_interface ${WIL_TARGET})
onnxruntime_add_include_to_target(onnxruntime_common date::date ${WIL_TARGET})
target_include_directories(onnxruntime_common
PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}
# propagate include directories of dependencies that are part of public interface
@ -208,7 +208,7 @@ if (ARM64 OR ARM OR X86 OR X64 OR X86_64)
# Its functionality in detecting x86 cpu features are lacking, so is support for Windows.
if (CPUINFO_SUPPORTED)
onnxruntime_add_include_to_target(onnxruntime_common cpuinfo::cpuinfo)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES cpuinfo::cpuinfo cpuinfo::clog)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES cpuinfo::cpuinfo ${ONNXRUNTIME_CLOG_TARGET_NAME})
endif()
endif()
endif()

View file

@ -98,7 +98,7 @@ if (MSVC)
endif()
if(NOT MSVC)
target_compile_options(onnxruntime_graph PRIVATE "-Wno-parentheses")
target_compile_options(onnxruntime_graph PRIVATE "-Wno-parentheses" "-Wno-deprecated-declarations")
endif()
if (onnxruntime_ENABLE_TRAINING)
#TODO: the graph library should focus on ONNX IR, it shouldn't depend on math libraries like MKLML/OpenBlas

View file

@ -39,6 +39,9 @@ function(AddTest)
endif()
if (MSVC)
target_compile_options(${_UT_TARGET} PRIVATE "/wd6330")
#Abseil has a lot of C4127/C4324 warnings.
target_compile_options(${_UT_TARGET} PRIVATE "/wd4127")
target_compile_options(${_UT_TARGET} PRIVATE "/wd4324")
endif()
set_target_properties(${_UT_TARGET} PROPERTIES FOLDER "ONNXRuntimeTest")
@ -60,7 +63,7 @@ function(AddTest)
target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} GTest::gtest GTest::gmock ${onnxruntime_EXTERNAL_LIBRARIES})
endif()
onnxruntime_add_include_to_target(${_UT_TARGET} date_interface flatbuffers::flatbuffers)
onnxruntime_add_include_to_target(${_UT_TARGET} date::date flatbuffers::flatbuffers)
target_include_directories(${_UT_TARGET} PRIVATE ${TEST_INC_DIR})
if (onnxruntime_USE_CUDA)
target_include_directories(${_UT_TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${onnxruntime_CUDNN_HOME}/include)
@ -146,7 +149,7 @@ function(AddTest)
else()
target_link_libraries(${_UT_TARGET}_xc PRIVATE ${_UT_LIBS} GTest::gtest GTest::gmock ${onnxruntime_EXTERNAL_LIBRARIES})
endif()
onnxruntime_add_include_to_target(${_UT_TARGET}_xc date_interface flatbuffers::flatbuffers)
onnxruntime_add_include_to_target(${_UT_TARGET}_xc date::date flatbuffers::flatbuffers)
target_include_directories(${_UT_TARGET}_xc PRIVATE ${TEST_INC_DIR})
get_target_property(${_UT_TARGET}_DEFS ${_UT_TARGET} COMPILE_DEFINITIONS)
target_compile_definitions(${_UT_TARGET}_xc PRIVATE ${_UT_TARGET}_DEFS)

View file

@ -49,7 +49,10 @@ function(add_winml_test)
if (_UT_DEPENDS)
add_dependencies(${_UT_TARGET} ${_UT_DEPENDS})
endif()
target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} gtest winml_google_test_lib ${onnxruntime_EXTERNAL_LIBRARIES} winml_lib_common onnxruntime windowsapp.lib)
target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} GTest::gtest winml_google_test_lib ${onnxruntime_EXTERNAL_LIBRARIES} winml_lib_common onnxruntime windowsapp.lib)
#Abseil has a lot of C4127/C4324 warnings.
target_compile_options(${_UT_TARGET} PRIVATE "/wd4127")
target_compile_options(${_UT_TARGET} PRIVATE "/wd4324")
target_compile_options(${_UT_TARGET} PRIVATE /wd5205) # workaround cppwinrt SDK bug https://github.com/microsoft/cppwinrt/issues/584
# if building inbox
@ -174,15 +177,18 @@ target_compile_options(winml_test_common PRIVATE /wd5205) # workaround cppwinrt
if (onnxruntime_WINML_NAMESPACE_OVERRIDE STREQUAL "Windows")
target_compile_definitions(winml_test_common PRIVATE "BUILD_INBOX=1")
endif()
#Abseil has a lot of C4127/C4324 warnings.
target_compile_options(winml_test_common PRIVATE "/wd4127")
target_compile_options(winml_test_common PRIVATE "/wd4324")
add_dependencies(winml_test_common
onnx
winml_api
winml_dll
)
onnxruntime_add_include_to_target(winml_test_common onnx_proto gtest ${PROTOBUF_LIB} ${WIL_TARGET} safeint_interface ${GSL_TARGET})
onnxruntime_add_include_to_target(winml_test_common onnx_proto GTest::gtest ${PROTOBUF_LIB} ${WIL_TARGET} safeint_interface ${GSL_TARGET})
onnxruntime_add_static_library(winml_google_test_lib ${WINML_TEST_SRC_DIR}/common/googletest/main.cpp)
onnxruntime_add_include_to_target(winml_google_test_lib gtest)
onnxruntime_add_include_to_target(winml_google_test_lib GTest::gtest)
set_winml_target_properties(winml_google_test_lib)
set_winml_target_properties(winml_test_common)

View file

@ -22,7 +22,6 @@
using namespace onnxruntime::logging;
using namespace onnxruntime::training;
using namespace google::protobuf::util;
using namespace onnxruntime::path_utils;
using namespace onnxruntime::test::training_session_test_utils;

View file

@ -17,7 +17,6 @@
using namespace onnxruntime::logging;
using namespace onnxruntime::training;
using namespace google::protobuf::util;
using namespace onnxruntime::path_utils;
using namespace onnxruntime::test::training_session_test_utils;

View file

@ -8,7 +8,6 @@
using namespace onnxruntime::logging;
using namespace onnxruntime::training;
using namespace google::protobuf::util;
using namespace onnxruntime::path_utils;
namespace onnxruntime {

View file

@ -95,42 +95,11 @@ def _build_aar(args):
exe_dir = os.path.join(intermediates_dir, "executables", build_config)
base_build_command = [sys.executable, BUILD_PY] + build_settings["build_params"] + ["--config=" + build_config]
header_files_path = ""
# Build and install protoc
protobuf_installation_script = os.path.join(
REPO_DIR,
"tools",
"ci_build",
"github",
"linux",
"docker",
"inference",
"x64",
"python",
"cpu",
"scripts",
"install_protobuf.sh",
)
subprocess.run(
[
protobuf_installation_script,
"-p",
os.path.join(build_dir, "protobuf"),
"-d",
os.path.join(REPO_DIR, "cmake", "deps.txt"),
],
shell=False,
check=True,
)
# Build binary for each ABI, one by one
for abi in build_settings["build_abis"]:
abi_build_dir = os.path.join(intermediates_dir, abi)
abi_build_command = [
*base_build_command,
"--android_abi=" + abi,
"--build_dir=" + abi_build_dir,
"--path_to_protoc_exe",
os.path.join(build_dir, "protobuf", "bin", "protoc"),
]
abi_build_command = [*base_build_command, "--android_abi=" + abi, "--build_dir=" + abi_build_dir]
if ops_config_path is not None:
abi_build_command += ["--include_ops_by_config=" + ops_config_path]

View file

@ -65,7 +65,9 @@ jobs:
- script: |
set -e -x
tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -p $(Build.BinariesDirectory)/installed -d cmake/deps.txt
rm -rf /tmp/scripts
cp -r tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts /tmp
/tmp/scripts/install_protobuf.sh -p $(Build.BinariesDirectory)/installed -d cmake/deps.txt
python3 tools/ci_build/build.py \
--config Release \
--android \

View file

@ -79,14 +79,6 @@ stages:
- template: "templates/use-android-ndk.yml"
# We build the host protoc to <ORT_ROOT>/protobuf_install
- script: |
/bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
$(Build.SourcesDirectory) \
$(Build.BinariesDirectory)/protobuf \
$(Build.SourcesDirectory)/protobuf_install
displayName: Build Host Protoc
- script: |
env | grep ANDROID
displayName: View Android ENVs
@ -102,7 +94,6 @@ stages:
--skip_submodule_sync \
--parallel \
--cmake_generator=Ninja \
--path_to_protoc_exe $(Build.SourcesDirectory)/protobuf_install/bin/protoc \
--build_java \
--skip_tests
displayName: CPU EP, Build
@ -172,14 +163,6 @@ stages:
- template: "templates/use-android-ndk.yml"
# We build the host protoc to <ORT_ROOT>/protobuf_install
- script: |
/bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
$(Build.SourcesDirectory) \
$(Build.BinariesDirectory)/protobuf \
$(Build.SourcesDirectory)/protobuf_install
displayName: Build Host Protoc
- script: |
env | grep ANDROID
displayName: View Android ENVs
@ -196,7 +179,6 @@ stages:
--parallel \
--use_nnapi \
--cmake_generator=Ninja \
--path_to_protoc_exe $(Build.SourcesDirectory)/protobuf_install/bin/protoc \
--build_java \
--skip_tests
displayName: NNAPI EP, Build
@ -372,14 +354,6 @@ stages:
- script: brew install coreutils ninja
displayName: Install coreutils and ninja
# We build the host protoc to <ORT_ROOT>/protobuf_install
- script: |
/bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
$(Build.SourcesDirectory) \
$(Build.BinariesDirectory)/protobuf \
$(Build.SourcesDirectory)/protobuf_install
displayName: Build Host Protoc
- script: /bin/bash tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh $(pwd)
# Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator
displayName: Build Minimal ORT with NNAPI and run tests
@ -462,7 +436,6 @@ stages:
--use_nnapi \
--build_shared_lib \
--cmake_generator=Ninja \
--path_to_protoc_exe $(Build.SourcesDirectory)/protobuf_install/bin/protoc \
--build_java \
--code_coverage
displayName: NNAPI EP, Build, Test, CodeCoverage on Android Emulator

View file

@ -37,21 +37,6 @@ jobs:
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
timeoutInMinutes: 150
steps:
- template: templates/mac-build-step-with-cache.yml
parameters:
WithCache: true
Today: $(TODAY)
AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
CacheDir: $(PROTO_CACHE_DIR)
ChangeEveryCommit: false
BuildStep:
- script: |
$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh \
-p $(Build.BinariesDirectory)/protobuf_install -d $(Build.SourcesDirectory)/cmake/deps.txt
displayName: Install protobuf
env:
CCACHE_DIR: $(PROTO_CACHE_DIR)
- template: templates/use-xcode-version.yml
- template: templates/mac-build-step-with-cache.yml
parameters:
@ -75,7 +60,6 @@ jobs:
--use_xcode \
--config RelWithDebInfo \
--build_apple_framework \
--path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
--parallel
displayName: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator
env:

View file

@ -11,7 +11,7 @@ steps:
packageType: upack
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0'
version: 1.0.78
version: 1.0.81
downloadPath: $(Build.BinariesDirectory)/deps
# The private ADO project
@ -22,7 +22,7 @@ steps:
packageType: upack
feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325'
definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a'
version: 1.0.78
version: 1.0.81
downloadPath: $(Build.BinariesDirectory)/deps
# You can add more ADO accounts at here.

View file

@ -158,7 +158,8 @@ CMD ["/bin/bash"]
ENV PATH ${DEVTOOLSET_ROOTPATH}/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev

View file

@ -1,101 +0,0 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# This script depends on python.exe, cmake.exe and Visual C++ spectre-mitigated libs.
# Please setup AGENT_TEMPDIRECTORY env variable before running this script
# This file is very similar to install_third_party_deps.ps1 expect this one only installs protobuf.
# This file is mainly for cross-compiling use cases.
param (
[string]$cpu_arch = "x64",
[string]$build_config = "RelWithDebInfo",
[string]$install_prefix = ".",
[switch]$use_cache
)
. "$PSScriptRoot\helpers.ps1"
$ort_src_root = (Get-Item $PSScriptRoot).parent.parent.parent.parent.FullName
Write-Host "ONNX Runtime src root: $ort_src_root"
$ErrorActionPreference = "Stop"
$Env:Path = "$install_prefix\bin;" + $env:Path
$Env:MSBUILDDISABLENODEREUSE=1
New-Item -Path "$install_prefix" -ItemType Directory -Force
# Setup compile flags
$compile_flags = @('/MP', '/guard:cf', '/Qspectre', '/DWIN32', '/D_WINDOWS', '/DWINVER=0x0A00', '/D_WIN32_WINNT=0x0A00', '/DNTDDI_VERSION=0x0A000000', '/W3')
$linker_flags=@('/guard:cf')
if ($use_cache) {
$debug_info_format = "/Z7"
}
else {
$debug_info_format = "/Zi"
}
if($build_config -eq 'Release'){
$compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL"
} elseif($build_config -eq 'RelWithDebInfo'){
$compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
} elseif($build_config -eq 'Debug'){
$compile_flags += "$debug_info_format", "/Ob0", "/Od", "/RTC1"
} elseif($build_config -eq 'MinSizeRel'){
$compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
}
# cmake args that applies to every 3rd-party library
[string[]]$cmake_extra_args="`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev"
if($cpu_arch -eq 'x86'){
$cmake_extra_args += "-A", "Win32", "-T", "host=x64"
$compile_flags += '/Qspectre'
$linker_flags += '/machine:x86'
} elseif($cpu_arch -eq 'x64') {
$linker_flags += '/machine:x64'
$compile_flags += '/Qspectre'
} elseif($cpu_arch -eq 'arm') {
$linker_flags += '/machine:ARM'
} elseif($cpu_arch -eq 'arm64') {
$linker_flags += '/machine:ARM64'
} elseif($cpu_arch -eq 'arm64ec') {
$linker_flags += '/machine:ARM64EC'
} else {
throw "$cpu_arch is not supported"
}
Write-Host $compile_flags
$cmake_extra_args += "-DCMAKE_CXX_STANDARD=17", "`"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`""
if ($use_cache) {
if ($build_config -eq 'RelWithDebInfo') {
$cmake_extra_args += "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=`"/MD /Z7 /O2 /Ob1 /DNDEBUG`""
}
elseif ($build_config -eq 'Debug') {
$cmake_extra_args += "-DCMAKE_CXX_FLAGS_DEBUG=`"/MDd /Z7 /Ob0 /Od /RTC1`""
}
}
$cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`""
# Find the full path of cmake.exe
$cmake_command = (Get-Command -CommandType Application cmake)[0]
$cmake_path = $cmake_command.Path
$msbuild_path = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\msbuild.exe"
if(-not (Test-Path $msbuild_path)){
$vshwere_path = Join-Path -Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"
if(-not (Test-Path $vshwere_path -PathType Leaf)){
$vshwere_path = Join-Path -Path ${env:ProgramFiles} "Microsoft Visual Studio\Installer\vswhere.exe"
}
$msbuild_path = &$vshwere_path -latest -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe | select-object -first 1
}
Write-Host "$msbuild_path"
Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path

View file

@ -1,4 +1,5 @@
#include "testPch.h"
#include <absl/strings/ascii.h>
#include "test/onnx/TestCase.h"
#include "test/onnx/heap_buffer.h"
#include "test/util/include/test/compare_ortvalue.h"
@ -387,7 +388,7 @@ std::string GetFullNameOfTest(ITestCase* testCase, winml::LearningModelDeviceKin
}
std::replace_if(
name.begin(), name.end(), [](char c) { return !google::protobuf::ascii_isalnum(c); }, '_'
name.begin(), name.end(), [](char c) { return !absl::ascii_isalnum(c); }, '_'
);
// Determine if test should be skipped, using the generic name (no CPU or GPU suffix yet).