From 99cfe8452a0590fe7c83130d178b59eb651e4fca Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:36:52 -0800 Subject: [PATCH] Upgrade emsdk version to v4.0.3 --- .gitmodules | 2 +- cmake/external/emsdk | 2 +- .../external/onnxruntime_external_deps.cmake | 18 +----- cmake/onnxruntime_webassembly.cmake | 55 ++++++++++++++++++ cmake/patches/emscripten/patch_3.1.74.tgz | Bin 4655 -> 0 bytes tools/ci_build/build.py | 2 +- .../templates/linux-wasm-ci.yml | 8 +-- 7 files changed, 64 insertions(+), 23 deletions(-) delete mode 100644 cmake/patches/emscripten/patch_3.1.74.tgz diff --git a/.gitmodules b/.gitmodules index 29ca8821f8..356447cbf7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,4 +7,4 @@ [submodule "cmake/external/emsdk"] path = cmake/external/emsdk url = https://github.com/emscripten-core/emsdk.git - branch = 3.1.59 + branch = 4.0.3 diff --git a/cmake/external/emsdk b/cmake/external/emsdk index d52c465201..127ce42cd5 160000 --- a/cmake/external/emsdk +++ b/cmake/external/emsdk @@ -1 +1 @@ -Subproject commit d52c46520124845b1e0e0525f2759299d840143f +Subproject commit 127ce42cd5f0aabe2d9b5d636041ccef7c66d165 diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 801d2292e0..a6c6929975 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -510,7 +510,7 @@ else() message("Setting pybind11_dep") set(pybind11_dep pybind11::pybind11) endif() - + endif() onnxruntime_fetchcontent_declare( onnx @@ -559,7 +559,7 @@ if (onnxruntime_USE_XNNPACK) find_library(xnnpack_LIBRARY NAMES XNNPACK) find_library(microkernels_prod_LIBRARY NAMES microkernels-prod) find_package(unofficial-pthreadpool CONFIG REQUIRED) - + target_include_directories(xnnpack INTERFACE "${XNNPACK_HDR}") set(XNNPACK_INCLUDE_DIR ${XNNPACK_DIR}/include) set(onnxruntime_EXTERNAL_LIBRARIES_XNNPACK ${xnnpack_LIBRARY} ${microkernels_prod_LIBRARY} unofficial::pthreadpool unofficial::pthreadpool_interface) @@ -657,20 +657,6 @@ if (onnxruntime_USE_WEBGPU) if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(DAWN_EMSCRIPTEN_TOOLCHAIN "${REPO_ROOT}/cmake/external/emsdk/upstream/emscripten" CACHE STRING "" FORCE) - - # Add the missing files from the emsdk installation - # - # For a "standard" emscripten build, the folder "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/" is not used. This is the - # reason why EMSDK installation does not include it. - # However, currently the WebGPU support in Emscripten is still being developed and the Dawn project is maintaining - # a fork of the Emscripten toolchain. As an extra build step, Dawn needs to generate some files using the file - # "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/gen_struct_info.py" from emscripten, which is missing in the emscripten - # installed by emsdk. - # - # We keep a copy of the missing file(s) in ${PROJECT_SOURCE_DIR}/patches/emscripten/, and now we extract them to the - # emscripten toolchain folder. - execute_process(COMMAND ${CMAKE_COMMAND} -E tar x "${PROJECT_SOURCE_DIR}/patches/emscripten/patch_3.1.74.tgz" - WORKING_DIRECTORY ${DAWN_EMSCRIPTEN_TOOLCHAIN}) else() if (onnxruntime_BUILD_DAWN_MONOLITHIC_LIBRARY) set(DAWN_BUILD_MONOLITHIC_LIBRARY ON CACHE BOOL "" FORCE) diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index 6cd05d1ad0..2c4cdebcf4 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -472,4 +472,59 @@ jsepDownload:_pp_") endif() set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME ${target_name} SUFFIX ".mjs") + + # + # The following POST_BUILD script is a workaround for enabling: + # - using onnxruntime-web with Multi-threading enabled when import from CDN + # - using onnxruntime-web when consumed in some frameworks like Vite + # + # In the use case mentioned above, the file name of the script may be changed. So we need to replace the line: + # `new Worker(new URL("ort-wasm-*.mjs", import.meta.url),` + # with + # `new Worker(new URL(import.meta.url),` + # + # This behavior is introduced in https://github.com/emscripten-core/emscripten/pull/22165. Since it's unlikely to be + # reverted, and there is no config to disable this behavior, we have to use a post-build script to workaround it. + # + + # Generate a script to do the post-build work + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/wasm_post_build.js " + const fs = require('fs'); + const path = require('path'); + + // node wasm_post_build.js + const mjsFilePath = process.argv[2]; + let contents = fs.readFileSync(mjsFilePath).toString(); + + const regex = 'new Worker\\\\(new URL\\\\(\".+?\", ?import\\\\.meta\\\\.url\\\\),'; + const matches = [...contents.matchAll(new RegExp(regex, 'g'))]; + if (matches.length !== 1) { + throw new Error( + `Unexpected number of matches for \"${regex}\" in \"${filepath}\": ${matches.length}.`, + ); + } + + // Replace the only occurrence. + contents = contents.replace( + new RegExp(regex), + `new Worker(new URL(import.meta.url),`, + ); + + fs.writeFileSync(mjsFilePath, contents); + " + ) + + find_program(NODE_EXECUTABLE node required) + if (NOT NODE_EXECUTABLE) + message(FATAL_ERROR "Node is required to run the post-build script") + endif() + + add_custom_command( + TARGET onnxruntime_webassembly + POST_BUILD + # Backup file at $.bak + COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$.bak" + COMMAND ${CMAKE_COMMAND} -E echo "Performing post-process for $" + COMMAND ${NODE_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/wasm_post_build.js" "$" + ) endif() diff --git a/cmake/patches/emscripten/patch_3.1.74.tgz b/cmake/patches/emscripten/patch_3.1.74.tgz deleted file mode 100644 index cfb52de6386538aede41508e829843fecc858433..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4655 zcmV+~6431*iwFS3=!Rzi1MNF$bK5wU`JAf!2eeZuN*$TDb53hlQ>C_*ta=>F<6}Bg zj*CN)ki>){H3aEnT>kI-4uAwH$#IfNPi<{j9hn614gl{4+TD8a-Jk94?bokg3;YXD z^Dlqwym8|CcYeUp)}tyeId0{-5vv& zhEX!@75QZIPx!r?F0Qj^I?Khg?Vac1WG2O6u7WID|Z% z3|)k`21iQx7k(7`<5-F!31ueo%Ty@2@H0P`B8~!?D5*Lk2P*G|C#FUCNr-KT?=*=n zWTvA0TBH*(jN>RtBeiv~d)ybJ1yDIoi!6|0pFx4MU(ju zbAPxyI5-}NNfb-q{?WsYjg6BT@QvCc!EjK;4n{KLC)BmE5nmS79<<1vp0A%JQ#QuR?}o*J#!{bM>ucxNBt9zyPfkT0Bd_oCWr-QV5RWU zY?nk(gTvCEql6#E(g$TrQqbvq2EvU>04$-bH@HH%{|L62hGFzYeEVaCe_Ni1$ULvz zM%XM(KMQHTYm^WiX2v5TuxVM|+U^JoV;i|+8-%W2Xzs^F6r+V-)fc46BwE>{W_~8a zl|9#>jB_uJ#%fh}kw-DD(_c@AgA021_#*cCN zA3fZCv)_Mv3@uvxAB+A)8ifM)hqpfkPcl~`EF_A{Gzf%R$RL_T@N%8x{*@MMXLHYc zdvNL6{nyx$!ho(x9^$FPH6$nB-C<`cDL;Z}Lr3!XmRd0p-g7>WMr{)va~(t9)9A6PS4 zLjcy#j$~1_OA_}2@ysrM4J@qL*MMEh@pMsCkTm&655fL};-#0Zxtg}Sh%lOHO>|jV zz@CDY6{K5TriexXR#Ik}dznSKY&nhSzAK)n4~Zk52t>=29#dS$(FbZ4N)mh*Nq{kq z5`aQdh_;LIAp})89O51fU?2)Z;~k{)1sJxF28hcD!a)wO$eM;C2JlQ^z{If}*>W-V zgHNEkQ;B{BA|lXYUxZO$BB~;jVtmawyuf3Utu{uM6kJ~U^F=JX_%1}?g;GI)2P>RI zhyXs4m}#3{C7}nFWRA`;dJ^HEot^c=o>}95Sv%@EcB>}}|7`q^geBZ)j`f+VWq6p6 z%n)$$ggo}P`^zsjB^>-aE+vH2MGR45qNjSpe?V<`rroLq+Gvhl0dwm#TAxAlFU>{_ zwjAQzDZ6gVH_c>M5Odsm%fiXCKywZu`}IP49dY5uh13%ui6nXP7Z90;Ek`@-jlZCY zKsfm0eE5)TwEqh|z?#v2!_kD_42PwHSE7YY3~f}V5(DP!Gra7r=B2`B@7{Ve{bkUG z)W#UhF$TSzJ#j5z01kA~;!#xgNnXSIL7I?n6ckJ$O3YmdwKf0{ zz`!_cIos);gDrrVq1EO-updsh>mUdW>zeS7LEzU_i#P(ggfShl(>~idFX0H`_POYR z*ZzxBf}-Vs7!K1H~>to{g_6~GpUM24tKqzlTIe^R;R>m>0~V>Hr>la8Z$E%juK|HSG?-o(jG;S0$|1$ z$QF(1kkqy7(mC@lC^N_;NS&frEkz`*Tu5MXHENAxvYsFV0)*FC*93p6yE=Qb|sLoxCkYM|}Yxn+4?{du;tzPsRx2C-F z#oTfzWHDSsTesVYs0$j|(ABjH${F&m<+a-GB<5yCf z+?U!FVt$W5KK}IaCm_~_f1E)V$kj`(@TK7M<}7!)Vf^A zFFiRA0MmseR(Tti=XsUVMXtSM7$h*W{o6;<4FFnrYXnR#Td}-M4GRxBJ zenXRVlPj|K8lI%M4EGwUejSZYj!%yIhlj(1w;sHm?w)wVgEyo5fe#OMNAD2w-DvN0 zf8hOg?DY?az54+Jwc+l`J;$CN9*&Mqy#3L^+uzUCdol45l#K!xZBZV}IQQ z{~(eA4fD8y?8)T9yYMq=UM3kzh-w0~7KRd;6RnVH7RxJ-A{zi}w{4p?4=a;qe6aSC zCU9w5u-~x=Mm#D$7^Ikn>P%SKPcw9h`n*q6dN&ChN(lOEP6FKdCoE$^-r%BR$RW(> zon!^OaJGzZ*&d>4+%+bVqP7~&(uxD}b#WLDNtLeU)!1`BBsu~lf8ol60NVy6h*MPq z(Pqt98pL5+oz*k^sMy(nFF<^>l0}k88G^BYv**@`Rj7^oBE$E^-iLufj!jXsJaWHMjxq)Hb>wL&1TqP(?Jx{;$IsnE5Mc&iEhr`1`n`=f?fofGw zAEQUAQFl$+s0|3rv!QrON1h->MFMy}wVFy&CVl2e4AxiWs>W~?^HzA4)(js!c&G`z zSqZs_{XlxEm_WwWSPaKSFsHptqmHiJ@s_@=-L7Tf8YM&AD=Sx<%WNvC zh~LU#KV(k&!!t)`0g0-vz*5GX1?y7-SWK+ZVSrDVtdZqHW_|>77?1*CT;zlWY&9Bg zea8V60L33L>6KYRW-4U5!gmL!>oTroPRo(^;Z)ZR)^b`RTjI3tW^AKe_c1G^UF+EA zTvQ{j)lfIq!T-%|r*^^P_O?m}*CmR=mJ|HxEHQG%FXtsEwwFbaW)T~YrdpMnF*u36 z(gkX4zKlSj(uJ;f0q2?1M%xM#kp8@B3BVf8ArV#(oxpUYfYdGI!zaEVqs>C-=RVLt z4qP0fTA6ggoVPM(Ys>u^3@{LKI+}=!N1$N(>z2qW79qM0RdoOZ&ZlbkK!Jmv1eg z-vk?|D=i!Lq00uPgqUVhvppZ~)}|S+R|zuxC?;Wr_$Q}bWz|cJaeW3>3}1h-KqFfZ2AK>IJve3mEpe${ev{WP6m3 zKN_{?Z6F&F77({uU1Rv^AJxvYTTfV@t zenOl7gz3QQ|h@&jHbm4koqfV`w3jGL|reP-5sY4X9zO$b)r8yia%0VP4X zFL+C({HeTiwW&nI%rP?foXgyYB)8|Fv$Gh(BihgKb%>FtJ~OL!hEjEXb3&j%<}z{7 zrlfOx5BXlKCIjv=;B16%(*6Us=pQ9?p&+8YYd*trKDsck>=}hoW-UJ}sWrp1^Uur= zi_U-poLb7Y2vhx6($9h!9cJ9c-CoIEWsxi9sOg*dI|O@-(Kp*hb9f0RkCVbQyNkKg zk~v)YCH73?zkxyC+ppnq6VsAcFV^VL-Lye=1cM9QEU=4o4~NO=gi}-H;1Ju`D6$PS zSN*WbC0*urxV{ z?%GVau%+G&-i?m_{^~`Kt_^W*4dS{i!Y%RRs;Fj8WD0Z?>4I-q>HQ_E3=#9CjQRYY z;}ozKd}TwqMqRthoRnt)dzq>Z1I;(;>s$q+|`9Qz=H(^_0@kc36BN=y8`r5+mFkVob_Y@1Z_7MrMOF zjyYgAj=D@$o$?YW1z2T_R$y9hBwt@QecF&C(YTMQ7Z%T`oZ#x}YcQG{pazw7`22bJ zDl7FSnLnNLZKM?mF|S<5LaCQ+MIve;YuELkl}+=f<2>?pR*+Z>a^I+XXRd-i>5!c&$aDQv|r lShNeeZUNZ-)Bg|BkLSnpeh3e*x;Za9{vJ000!_0KotN diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 36de85ea2e..c234a69a73 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -488,7 +488,7 @@ def parse_arguments(): # WebAssembly build parser.add_argument("--build_wasm", action="store_true", help="Build for WebAssembly") parser.add_argument("--build_wasm_static_lib", action="store_true", help="Build for WebAssembly static library") - parser.add_argument("--emsdk_version", default="3.1.59", help="Specify version of emsdk") + parser.add_argument("--emsdk_version", default="4.0.3", help="Specify version of emsdk") parser.add_argument("--enable_wasm_simd", action="store_true", help="Enable WebAssembly SIMD") parser.add_argument("--enable_wasm_threads", action="store_true", help="Enable WebAssembly multi-threads support") diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml index c0bc960521..6727b1534e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml @@ -82,15 +82,15 @@ jobs: - script: | set -ex cd '$(Build.SourcesDirectory)/cmake/external/emsdk' - ./emsdk install 3.1.59 ccache-git-emscripten-64bit - ./emsdk activate 3.1.59 ccache-git-emscripten-64bit + ./emsdk install 4.0.3 ccache-git-emscripten-64bit + ./emsdk activate 4.0.3 ccache-git-emscripten-64bit displayName: 'emsdk install and activate ccache for emscripten' - ${{if eq(parameters.WithCache, false)}}: - script: | set -ex cd '$(Build.SourcesDirectory)/cmake/external/emsdk' - ./emsdk install 3.1.59 - ./emsdk activate 3.1.59 + ./emsdk install 4.0.3 + ./emsdk activate 4.0.3 displayName: 'emsdk install and activate ccache for emscripten' - template: build-linux-wasm-step.yml