diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5a0cfd7fce..cfede4e6a5 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -147,6 +147,7 @@ option(onnxruntime_USE_MPI "Build with MPI support" OFF) # build WebAssembly option(onnxruntime_BUILD_WEBASSEMBLY "Enable this option to create WebAssembly byte codes" OFF) +option(onnxruntime_ENABLE_WEBASSEMBLY_THREADS "Enable this option to create WebAssembly byte codes with multi-threads support" OFF) option(onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING "Enable this option to turn on exception catching" OFF) # Enable bitcode for iOS @@ -263,6 +264,16 @@ if (NOT MSVC AND NOT onnxruntime_ENABLE_BITCODE) string(APPEND CMAKE_C_FLAGS " -ffunction-sections -fdata-sections") endif() +# Build WebAssembly source map on debug build. +if (onnxruntime_BUILD_WEBASSEMBLY) + string(APPEND CMAKE_CXX_FLAGS_DEBUG " -g4 --source-map-base http://localhost:8000/") + + # Build WebAssembly with multi-threads support. + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + string(APPEND CMAKE_CXX_FLAGS " -pthread") + endif() +endif() + # ORT build with as much excluded as possible. Supports ORT flatbuffers models only. if (onnxruntime_MINIMAL_BUILD) add_compile_definitions(ORT_MINIMAL_BUILD) @@ -564,7 +575,8 @@ if (onnxruntime_BUILD_UNIT_TESTS) if(NOT TARGET GTest::gtest) message("Use gtest from submodule") - # WebAssembly doesn't support threading properly yet. + # WebAssembly threading support in Node.js is still an experimental feature and + # not working properly with googletest suite. if (onnxruntime_BUILD_WEBASSEMBLY) set(gtest_disable_pthreads ON) endif() @@ -619,7 +631,7 @@ if(onnxruntime_BUILD_BENCHMARKS) endif() endif() -if(NOT WIN32 AND NOT onnxruntime_PREFER_SYSTEM_LIB AND NOT onnxruntime_BUILD_WEBASSEMBLY) +if(NOT WIN32 AND NOT onnxruntime_PREFER_SYSTEM_LIB) add_subdirectory(${PROJECT_SOURCE_DIR}/external/nsync EXCLUDE_FROM_ALL) endif() # External dependencies @@ -775,7 +787,7 @@ function(onnxruntime_add_shared_library target_name) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") - elseif(NOT onnxruntime_BUILD_WEBASSEMBLY) + else() target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") endif() @@ -795,7 +807,7 @@ function(onnxruntime_add_shared_library_module target_name) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") - elseif(NOT onnxruntime_BUILD_WEBASSEMBLY) + else() target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") endif() @@ -815,7 +827,7 @@ function(onnxruntime_add_executable target_name) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") - elseif(NOT onnxruntime_BUILD_WEBASSEMBLY) + else() target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") endif() @@ -1463,7 +1475,7 @@ foreach(target_name onnxruntime_common onnxruntime_graph onnxruntime_framework o target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") set_target_properties(${target_name} PROPERTIES VS_CA_EXCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") - elseif(NOT onnxruntime_BUILD_WEBASSEMBLY) + else() target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(${target_name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") endif() @@ -1475,7 +1487,7 @@ foreach(provider_name ${ONNXRUNTIME_PROVIDER_NAMES}) if (MSVC) target_compile_options(onnxruntime_providers_${provider_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") target_compile_options(onnxruntime_providers_${provider_name} PRIVATE "$<$:SHELL:--compiler-options /sdl>" "$<$>:/sdl>") - elseif(NOT onnxruntime_BUILD_WEBASSEMBLY) + else() target_compile_definitions(onnxruntime_providers_${provider_name} PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(onnxruntime_providers_${provider_name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") endif() @@ -1495,7 +1507,7 @@ endif() if(WIN32) list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${SYS_PATH_LIB} Shlwapi) list(APPEND onnxruntime_EXTERNAL_LIBRARIES debug Dbghelp) -elseif(NOT onnxruntime_BUILD_WEBASSEMBLY) +else() list(APPEND onnxruntime_EXTERNAL_LIBRARIES nsync_cpp) list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${CMAKE_DL_LIBS} Threads::Threads) endif() @@ -1586,6 +1598,10 @@ if (onnxruntime_BUILD_WEBASSEMBLY) string(APPEND CMAKE_CXX_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") endif() + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + string(APPEND CMAKE_CXX_FLAGS " -s USE_PTHREADS=1 -Wno-pthreads-mem-growth") + endif() + message(STATUS "WebAssembly Build is enabled") include(onnxruntime_webassembly.cmake) endif() diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index fd65893355..222eab1ec6 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -86,12 +86,6 @@ file(GLOB onnxruntime_common_src CONFIGURE_DEPENDS ${onnxruntime_common_src_patterns} ) -if (onnxruntime_BUILD_WEBASSEMBLY) - list(REMOVE_ITEM onnxruntime_common_src - "${ONNXRUNTIME_ROOT}/core/platform/posix/ort_mutex.cc" - ) -endif() - source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_common_src}) add_library(onnxruntime_common ${onnxruntime_common_src}) @@ -130,7 +124,7 @@ target_include_directories(onnxruntime_common target_link_libraries(onnxruntime_common Boost::mp11) -if(NOT WIN32 AND NOT onnxruntime_BUILD_WEBASSEMBLY) +if(NOT WIN32) target_include_directories(onnxruntime_common PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") endif() diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index fc85c40885..1eb5e5a5f2 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -127,10 +127,21 @@ function(AddTest) xctest_add_test(xctest.${_UT_TARGET} ${_UT_TARGET}_xc) else() - add_test(NAME ${_UT_TARGET} - COMMAND ${_UT_TARGET} ${TEST_ARGS} - WORKING_DIRECTORY $ - ) + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + find_program(NODE_EXECUTABLE node required) + if (NOT NODE_EXECUTABLE) + message(FATAL_ERROR "Node is required for unit tests") + endif() + add_test(NAME ${_UT_TARGET} + COMMAND ${NODE_EXECUTABLE} --experimental-wasm-threads --experimental-wasm-bulk-memory ${_UT_TARGET}.js ${TEST_ARGS} + WORKING_DIRECTORY $ + ) + else() + add_test(NAME ${_UT_TARGET} + COMMAND ${_UT_TARGET} ${TEST_ARGS} + WORKING_DIRECTORY $ + ) + endif() endif() endfunction(AddTest) @@ -527,7 +538,7 @@ add_library(onnxruntime_test_utils ${onnxruntime_test_utils_src}) if(MSVC) target_compile_options(onnxruntime_test_utils PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") -elseif (NOT onnxruntime_BUILD_WEBASSEMBLY) +else() target_compile_definitions(onnxruntime_test_utils PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(onnxruntime_test_utils PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") @@ -556,7 +567,7 @@ add_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) if(MSVC) target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") -elseif (NOT onnxruntime_BUILD_WEBASSEMBLY) +else() target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") @@ -606,17 +617,25 @@ if(WIN32) endif() if (onnxruntime_BUILD_WEBASSEMBLY) - # blacklist for WebAssembly for now. - # exclude thread test since WebAssembly doesn't support multi-threads properly yet. - list(REMOVE_ITEM all_tests - "${TEST_SRC_DIR}/framework/execution_frame_test.cc" - "${TEST_SRC_DIR}/framework/inference_session_test.cc" - "${TEST_SRC_DIR}/platform/barrier_test.cc" - "${TEST_SRC_DIR}/platform/threadpool_test.cc" - "${TEST_SRC_DIR}/providers/cpu/controlflow/loop_test.cc" - "${TEST_SRC_DIR}/providers/cpu/nn/string_normalizer_test.cc" - "${TEST_SRC_DIR}/providers/memcpy_test.cc" - ) + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + # WebAssembly threading support in node is an experimental feature yet + # and that makes some intensive threadpool tests fail randomly. + # Will enable this test when node.js releases a stable version supporting multi-threads. + list(REMOVE_ITEM all_tests + "${TEST_SRC_DIR}/platform/threadpool_test.cc" + "${TEST_SRC_DIR}/providers/cpu/nn/string_normalizer_test.cc" + ) + else() + list(REMOVE_ITEM all_tests + "${TEST_SRC_DIR}/framework/execution_frame_test.cc" + "${TEST_SRC_DIR}/framework/inference_session_test.cc" + "${TEST_SRC_DIR}/platform/barrier_test.cc" + "${TEST_SRC_DIR}/platform/threadpool_test.cc" + "${TEST_SRC_DIR}/providers/cpu/controlflow/loop_test.cc" + "${TEST_SRC_DIR}/providers/cpu/nn/string_normalizer_test.cc" + "${TEST_SRC_DIR}/providers/memcpy_test.cc" + ) + endif() endif() AddTest( @@ -653,7 +672,11 @@ if (onnxruntime_USE_ROCM) target_include_directories(onnxruntime_test_all PRIVATE ${onnxruntime_ROCM_HOME}/hipfft/include ${onnxruntime_ROCM_HOME}/include ${onnxruntime_ROCM_HOME}/hiprand/include ${onnxruntime_ROCM_HOME}/rocrand/include ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) endif() if (onnxruntime_BUILD_WEBASSEMBLY) - set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata") + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") + else() + set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s EXIT_RUNTIME=1") + endif() endif() set(test_data_target onnxruntime_test_all) @@ -864,7 +887,7 @@ if (onnxruntime_BUILD_SHARED_LIB) onnxruntime onnxruntime_flatbuffers onnx_test_data_proto ${onnxruntime_EXTERNAL_LIBRARIES} ${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS}) - if(NOT WIN32 AND NOT onnxruntime_BUILD_WEBASSEMBLY) + if(NOT WIN32) list(APPEND onnxruntime_perf_test_libs nsync_cpp) endif() if (CMAKE_SYSTEM_NAME STREQUAL "Android") @@ -904,7 +927,7 @@ if (onnxruntime_BUILD_SHARED_LIB) ################################################################# # test inference using shared lib set(onnxruntime_shared_lib_test_LIBS onnxruntime_mocked_allocator onnxruntime_test_utils onnxruntime_common onnx_proto) - if(NOT WIN32 AND NOT onnxruntime_BUILD_WEBASSEMBLY) + if(NOT WIN32) list(APPEND onnxruntime_shared_lib_test_LIBS nsync_cpp) endif() if (onnxruntime_USE_CUDA) @@ -998,7 +1021,7 @@ endif() target_include_directories(onnxruntime_mlas_test PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR}) set(onnxruntime_mlas_test_libs GTest::gtest GTest::gmock onnxruntime_mlas onnxruntime_common) -if(NOT WIN32 AND NOT onnxruntime_BUILD_WEBASSEMBLY) +if(NOT WIN32) list(APPEND onnxruntime_mlas_test_libs nsync_cpp ${CMAKE_DL_LIBS}) endif() if (onnxruntime_USE_OPENMP) @@ -1014,7 +1037,11 @@ if (onnxruntime_LINK_LIBATOMIC) endif() set_target_properties(onnxruntime_mlas_test PROPERTIES FOLDER "ONNXRuntimeTest") if (onnxruntime_BUILD_WEBASSEMBLY) - set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1") + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") + else() + set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1") + endif() endif() add_library(custom_op_library SHARED ${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc) diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index 7b1a04b6c5..be7b7b7fe5 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -11,18 +11,19 @@ add_executable(onnxruntime_wasm ${onnxruntime_wasm_src} ) -add_compile_definitions( - MLAS_NO_ONNXRUNTIME_THREADPOOL -) +if (NOT onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + add_compile_definitions( + MLAS_NO_ONNXRUNTIME_THREADPOOL + ) -string(APPEND CMAKE_CXX_FLAGS_DEBUG " -g4 --source-map-base http://localhost:8000/") + # Override re2 compiler options to remove -pthread + set_property(TARGET re2 PROPERTY COMPILE_OPTIONS ) +endif() target_compile_options(onnx PRIVATE -Wno-unused-parameter -Wno-unused-variable) -# Override re2 compiler options to remove -pthread -set_property(TARGET re2 PROPERTY COMPILE_OPTIONS ) - target_link_libraries(onnxruntime_wasm PRIVATE + nsync_cpp protobuf::libprotobuf-lite onnx onnx_proto @@ -42,21 +43,25 @@ set(EXTRA_EXPORTED_RUNTIME_METHODS "['stackAlloc','stackRestore','stackSave','UT set_target_properties(onnxruntime_wasm PROPERTIES LINK_FLAGS " \ -s \"EXTRA_EXPORTED_RUNTIME_METHODS=${EXTRA_EXPORTED_RUNTIME_METHODS}\" \ - -s EXPORT_NAME=ort \ -s WASM=1 \ -s NO_EXIT_RUNTIME=0 \ -s ALLOW_MEMORY_GROWTH=1 \ - -s SAFE_HEAP=0 \ -s MODULARIZE=1 \ - -s SAFE_HEAP_LOG=0 \ - -s STACK_OVERFLOW_CHECK=0 \ -s EXPORT_ALL=0 \ -s LLD_REPORT_UNDEFINED \ -s VERBOSE=0 \ -s NO_FILESYSTEM=1 \ --no-entry") + if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set_property(TARGET onnxruntime_wasm APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=1 -s DEMANGLE_SUPPORT=1") + set_property(TARGET onnxruntime_wasm APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=2 -s SAFE_HEAP=1 -s STACK_OVERFLOW_CHECK=1 -s DEMANGLE_SUPPORT=1") else() - set_property(TARGET onnxruntime_wasm APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s DEMANGLE_SUPPORT=0") + set_property(TARGET onnxruntime_wasm APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s SAFE_HEAP=0 -s STACK_OVERFLOW_CHECK=0 -s DEMANGLE_SUPPORT=0") +endif() + +if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + set_property(TARGET onnxruntime_wasm APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=onnxWasmThreadsBindingJs -s USE_PTHREADS=1") + set_target_properties(onnxruntime_wasm PROPERTIES OUTPUT_NAME "onnxruntime_wasm_threads") +else() + set_property(TARGET onnxruntime_wasm APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=onnxWasmBindingJs") endif() diff --git a/include/onnxruntime/core/platform/ort_mutex.h b/include/onnxruntime/core/platform/ort_mutex.h index 187dcdcca1..e24665f514 100644 --- a/include/onnxruntime/core/platform/ort_mutex.h +++ b/include/onnxruntime/core/platform/ort_mutex.h @@ -101,13 +101,6 @@ std::cv_status OrtCondVar::wait_for(std::unique_lock& cond_mutex, return steady_clock::now() - steady_now < rel_time ? std::cv_status::no_timeout : std::cv_status::timeout; } } // namespace onnxruntime -#elif defined(__wasm__) -#include -#include -namespace onnxruntime { -using OrtMutex = std::mutex; -using OrtCondVar = std::condition_variable; -} // namespace onnxruntime #else #include "nsync.h" #include //for unique_lock diff --git a/onnxruntime/wasm/api.cc b/onnxruntime/wasm/api.cc index c3ba4eedda..94c126f992 100644 --- a/onnxruntime/wasm/api.cc +++ b/onnxruntime/wasm/api.cc @@ -22,8 +22,10 @@ Ort::Session* OrtCreateSession(void* data, size_t data_length) { Ort::SessionOptions session_options; session_options.SetLogId("onnxruntime"); - // disable thread pool for now since not all major browsers support WebAssembly threading. +#if !defined(__EMSCRIPTEN_PTHREADS__) + // must disable thread pool when WebAssembly multi-threads support is disabled. session_options.SetIntraOpNumThreads(1); +#endif return new Ort::Session(*g_env, data, data_length, session_options); } diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5e44775cbe..36a0e4e18c 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -332,6 +332,9 @@ def parse_arguments(): parser.add_argument( "--disable_wasm_exception_catching", action='store_true', help="Disable exception catching in WebAssembly.") + parser.add_argument( + "--enable_wasm_threads", action='store_true', + help="Enable WebAssembly multi-threads support") # Arguments needed by CI parser.add_argument( @@ -721,6 +724,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_BUILD_WEBASSEMBLY=" + ("ON" if args.build_wasm else "OFF"), "-Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING=" + ("OFF" if args.disable_wasm_exception_catching else "ON"), + "-Donnxruntime_ENABLE_WEBASSEMBLY_THREADS=" + ("ON" if args.enable_wasm_threads else "OFF"), ] if acl_home and os.path.exists(acl_home):