From 0457fd0b4078fbca07d9506aa1bcda2a839afc81 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Mon, 8 May 2023 16:49:47 -0700 Subject: [PATCH] upgrade emsdk to 3.1.37 (#15817) ### Description upgrade emsdk to 3.1.37 WIP branch to debug the mystery memory issue in web assembly multi-thread build. --- .gitmodules | 2 +- cgmanifests/generated/cgmanifest.json | 50 +++++++++++++++++++ cmake/adjust_global_compile_flags.cmake | 6 +-- cmake/external/emsdk | 2 +- cmake/onnxruntime_unittests.cmake | 15 ++++-- cmake/onnxruntime_webassembly.cmake | 4 +- onnxruntime/core/platform/posix/env.cc | 22 +++++++- tools/ci_build/build.py | 5 +- .../azure-pipelines/templates/web-ci.yml | 2 +- .../azure-pipelines/templates/win-wasm-ci.yml | 6 +-- 10 files changed, 95 insertions(+), 19 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1319cfacc1..19f55e6c7b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,7 +10,7 @@ [submodule "cmake/external/emsdk"] path = cmake/external/emsdk url = https://github.com/emscripten-core/emsdk.git - branch = 3.1.32 + branch = 3.1.37 [submodule "cmake/external/onnxruntime-extensions"] path = cmake/external/onnxruntime-extensions url = https://github.com/microsoft/onnxruntime-extensions.git diff --git a/cgmanifests/generated/cgmanifest.json b/cgmanifests/generated/cgmanifest.json index 35a7d831f6..18da201261 100644 --- a/cgmanifests/generated/cgmanifest.json +++ b/cgmanifests/generated/cgmanifest.json @@ -78,6 +78,56 @@ "comments": "manylinux dependency LIBXCRYPT" } }, + { + "component": { + "type": "git", + "git": { + "commitHash": "d10b27fe37736d2944630ecd7557cefa95cf87c9", + "repositoryUrl": "https://gitlab.com/libeigen/eigen.git" + }, + "comments": "git submodule at cmake/external/eigen" + } + }, + { + "component": { + "type": "git", + "git": { + "commitHash": "b113f24842c6e97fe3e352084db09a6e278593ae", + "repositoryUrl": "https://github.com/emscripten-core/emsdk.git" + }, + "comments": "git submodule at cmake/external/emsdk" + } + }, + { + "component": { + "type": "git", + "git": { + "commitHash": "7a2ed51a6b682a83e345ff49fc4cfd7ca47550db", + "repositoryUrl": "https://github.com/google/libprotobuf-mutator.git" + }, + "comments": "git submodule at cmake/external/libprotobuf-mutator" + } + }, + { + "component": { + "type": "git", + "git": { + "commitHash": "9b7bca2a723ff94edcd007d93b5d0cf1838591dc", + "repositoryUrl": "https://github.com/onnx/onnx.git" + }, + "comments": "git submodule at cmake/external/onnx" + } + }, + { + "component": { + "type": "git", + "git": { + "commitHash": "81e7799c69044c745239202085eb0a98f102937b", + "repositoryUrl": "https://github.com/microsoft/onnxruntime-extensions.git" + }, + "comments": "git submodule at cmake/external/onnxruntime-extensions" + } + }, { "component": { "type": "git", diff --git a/cmake/adjust_global_compile_flags.cmake b/cmake/adjust_global_compile_flags.cmake index d4421a6211..92f4a4eaf2 100644 --- a/cmake/adjust_global_compile_flags.cmake +++ b/cmake/adjust_global_compile_flags.cmake @@ -51,10 +51,8 @@ if (onnxruntime_BUILD_WEBASSEMBLY) # Build WebAssembly with multi-threads support. if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - string(APPEND CMAKE_C_FLAGS " -pthread") - string(APPEND CMAKE_CXX_FLAGS " -pthread") - string(APPEND CMAKE_C_FLAGS " -s USE_PTHREADS=1 -Wno-pthreads-mem-growth") - string(APPEND CMAKE_CXX_FLAGS " -s USE_PTHREADS=1 -Wno-pthreads-mem-growth") + string(APPEND CMAKE_C_FLAGS " -pthread -Wno-pthreads-mem-growth") + string(APPEND CMAKE_CXX_FLAGS " -pthread -Wno-pthreads-mem-growth") endif() endif() diff --git a/cmake/external/emsdk b/cmake/external/emsdk index 0ab19024f0..b113f24842 160000 --- a/cmake/external/emsdk +++ b/cmake/external/emsdk @@ -1 +1 @@ -Subproject commit 0ab19024f08c6673a713e454ef8bd95e174c807f +Subproject commit b113f24842c6e97fe3e352084db09a6e278593ae diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 9347be180d..a323eae850 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -848,13 +848,20 @@ if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) endif() if (onnxruntime_BUILD_WEBASSEMBLY) set_target_properties(onnxruntime_test_all PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/onnxruntime_test_all_adapter.js) - set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 --pre-js \"${TEST_SRC_DIR}/wasm/onnxruntime_test_all_adapter.js\" -s \"EXPORTED_RUNTIME_METHODS=['FS']\" --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s EXIT_RUNTIME=1") + set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s STACK_SIZE=5242880 -s ALLOW_MEMORY_GROWTH=1 --pre-js \"${TEST_SRC_DIR}/wasm/onnxruntime_test_all_adapter.js\" -s \"EXPORTED_RUNTIME_METHODS=['FS']\" --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s EXIT_RUNTIME=1 -s DEMANGLE_SUPPORT=1") if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - set_property(TARGET onnxruntime_test_all APPEND_STRING PROPERTY LINK_FLAGS " -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1") + set_property(TARGET onnxruntime_test_all APPEND_STRING PROPERTY LINK_FLAGS " -s DEFAULT_PTHREAD_STACK_SIZE=131072 -s PROXY_TO_PTHREAD=1") endif() if (onnxruntime_USE_JSEP) set_property(TARGET onnxruntime_test_all APPEND_STRING PROPERTY LINK_FLAGS " --pre-js \"${ONNXRUNTIME_ROOT}/wasm/js_internal_api.js\"") endif() + + ### + ### if you want to investigate or debug a test failure in onnxruntime_test_all, replace the following line. + ### those flags slow down the CI test significantly, so we don't use them by default. + ### + # set_property(TARGET onnxruntime_test_all APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=2 -s SAFE_HEAP=1 -s STACK_OVERFLOW_CHECK=2") + set_property(TARGET onnxruntime_test_all APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s SAFE_HEAP=0 -s STACK_OVERFLOW_CHECK=1") endif() if (onnxruntime_ENABLE_ATEN) @@ -1005,7 +1012,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") endif() if (onnxruntime_BUILD_WEBASSEMBLY) if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") + set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") else() set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1") endif() @@ -1349,7 +1356,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) set_target_properties(onnxruntime_mlas_test PROPERTIES FOLDER "ONNXRuntimeTest") if (onnxruntime_BUILD_WEBASSEMBLY) if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") + set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") else() set_target_properties(onnxruntime_mlas_test PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1") endif() diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index 193315f541..df41e2fd14 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -242,7 +242,7 @@ else() target_link_options(onnxruntime_webassembly PRIVATE "SHELL:-s ASSERTIONS=2" "SHELL:-s SAFE_HEAP=1" - "SHELL:-s STACK_OVERFLOW_CHECK=1" + "SHELL:-s STACK_OVERFLOW_CHECK=2" "SHELL:-s DEMANGLE_SUPPORT=1" ) else() @@ -265,7 +265,7 @@ else() if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) target_link_options(onnxruntime_webassembly PRIVATE "SHELL:-s EXPORT_NAME=ortWasmThreaded" - "SHELL:-s USE_PTHREADS=1" + "SHELL:-s DEFAULT_PTHREAD_STACK_SIZE=131072" ) if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd-threaded") diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc index 14217a126d..bcc21c5f03 100644 --- a/onnxruntime/core/platform/posix/env.cc +++ b/onnxruntime/core/platform/posix/env.cc @@ -191,8 +191,26 @@ class PosixThread : public EnvThread { auto [err_no, err_msg] = GetSystemError(); ORT_THROW("pthread_attr_init failed, error code: ", err_no, " error msg: ", err_msg); } - if (thread_options.stack_size > 0) { - s = pthread_attr_setstacksize(&attr, thread_options.stack_size); + + size_t stack_size = thread_options.stack_size; +#if defined(__wasm__) + // emscripten 3.1.37 has a bug which does not take build flags 'STACK_SIZE' or 'DEFAULT_PTHREAD_STACK_SIZE'. + // the pthread stack size will always be 64kB, which is insufficient to run some kernels. + // we set the stack_size to a bigger value + // + // https://github.com/emscripten-core/emscripten/issues/19302 + // + // TODO: once this issue is fixed by emscripten's new release, remove this code. + // future changes to DEFAULT_PTHREAD_STACK_SIZE will be in the following files + // - cmake/onnxruntime_unittests.cmake (target onnxruntime_test_all) + // - cmake/onnxruntime_webassembly.cmake (target onnxruntime_webassembly) + // + if (stack_size == 0) { + stack_size = 131072; + } +#endif + if (stack_size > 0) { + s = pthread_attr_setstacksize(&attr, stack_size); if (s != 0) { auto [err_no, err_msg] = GetSystemError(); ORT_THROW("pthread_attr_setstacksize failed, error code: ", err_no, " error msg: ", err_msg); diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index ad4befba14..737a3ee38c 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -396,7 +396,7 @@ def parse_arguments(): # WebAssembly build parser.add_argument("--build_wasm", action="store_true", help="Build for WebAssembly") parser.add_argument("--build_wasm_static_lib", action="store_true", help="Build for WebAssembly static library") - parser.add_argument("--emsdk_version", default="3.1.19", help="Specify version of emsdk") + parser.add_argument("--emsdk_version", default="3.1.37", help="Specify version of emsdk") parser.add_argument("--enable_wasm_simd", action="store_true", help="Enable WebAssembly SIMD") parser.add_argument("--enable_wasm_threads", action="store_true", help="Enable WebAssembly multi-threads support") @@ -1245,6 +1245,9 @@ def generate_build_tree( add_default_definition(emscripten_settings, "MALLOC", args.wasm_malloc) add_default_definition(emscripten_settings, "MALLOC", "dlmalloc") + # set -s STACK_SIZE=5242880 + add_default_definition(emscripten_settings, "STACK_SIZE", "5242880") + if emscripten_settings: cmake_args += [f"-Donnxruntime_EMSCRIPTEN_SETTINGS={';'.join(emscripten_settings)}"] diff --git a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml index f860251737..0c672d4eda 100644 --- a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml @@ -81,7 +81,7 @@ stages: parameters: CommitOverride: true BuildConfig: 'Release' - ExtraBuildArgs: '--skip_tests --enable_wasm_api_exception_catching --disable_rtti --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}' + ExtraBuildArgs: '--target onnxruntime_webassembly --skip_tests --enable_wasm_api_exception_catching --disable_rtti --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}' PoolName: ${{ parameters.PoolName }} BuildJsep: ${{ parameters.BuildJsep }} diff --git a/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml index e6797b0737..c04c2b41a7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml @@ -33,7 +33,7 @@ jobs: variables: EnvSetupScript: setup_env.bat buildArch: x64 - CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --cmake_generator "MinGW Makefiles" --build_wasm --use_xnnpack --emsdk_version releases-upstream-4c3772879a04140298c3abde90962d5567b5e2fc-64bit ${{ parameters.ExtraBuildArgs }}' + CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --cmake_generator "MinGW Makefiles" --build_wasm --use_xnnpack --emsdk_version releases-7c905cfc1ca6699f6ccb288ae174902cfbdcf0a2-64bit ${{ parameters.ExtraBuildArgs }}' runCodesignValidationInjection: false timeoutInMinutes: ${{ parameters.TimeoutInMinutes }} workspace: @@ -88,10 +88,10 @@ jobs: arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_threads --enable_wasm_threads' workingDirectory: '$(Build.BinariesDirectory)' - task: PythonScript@0 - displayName: 'Build and test (node) (simd + threads)' + displayName: 'Build and test (browser) (simd + threads)' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_simd_threads --enable_wasm_simd --enable_wasm_threads' + arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_simd_threads --enable_wasm_simd --enable_wasm_threads --wasm_run_tests_in_browser' workingDirectory: '$(Build.BinariesDirectory)' - task: PythonScript@0 displayName: 'Build and test (node) (simd)'