From 822fe243d4ee6ff02390a3170eaf19d1de7da7de Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 16 Jan 2025 00:39:48 -0800 Subject: [PATCH] working... --- build_all.bat | 3 + cmake/onnxruntime_webassembly.cmake | 21 +- js/build_webgpu.bat | 79 ++++++++ js/web/lib/build-def.d.ts | 7 + js/web/lib/wasm/ep-webgpu/README.md | 11 + js/web/lib/wasm/session-options.ts | 39 ++-- js/web/lib/wasm/wasm-core-impl.ts | 112 +++++++---- js/web/lib/wasm/wasm-types.ts | 14 +- js/web/script/build.ts | 1 + onnxruntime/wasm/api.cc | 8 +- onnxruntime/wasm/api.h | 5 +- onnxruntime/wasm/js_post_js.js | 2 +- onnxruntime/wasm/js_post_js_64.js | 2 +- onnxruntime/wasm/pre-async.js | 142 +++++++++++++ onnxruntime/wasm/pre-jsep.js | 298 ++++++++++------------------ onnxruntime/wasm/pre.js | 15 +- tools/ci_build/build.py | 3 - 17 files changed, 487 insertions(+), 275 deletions(-) create mode 100644 build_all.bat create mode 100644 js/build_webgpu.bat create mode 100644 js/web/lib/wasm/ep-webgpu/README.md create mode 100644 onnxruntime/wasm/pre-async.js diff --git a/build_all.bat b/build_all.bat new file mode 100644 index 0000000000..2b90fd9cda --- /dev/null +++ b/build_all.bat @@ -0,0 +1,3 @@ +build --config Release --build_wasm --enable_wasm_simd --enable_wasm_threads --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON --target onnxruntime_webassembly --skip_tests --enable_wasm_api_exception_catching --disable_rtti --build_dir ./build_wasm_inferencing +build --config Release --build_wasm --enable_wasm_simd --enable_wasm_threads --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON --target onnxruntime_webassembly --skip_tests --enable_wasm_api_exception_catching --disable_rtti --build_dir ./build_wasm_inferencing_jsep --use_jsep --use_webnn +build --config Release --build_wasm --enable_wasm_simd --enable_wasm_threads --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON --target onnxruntime_webassembly --skip_tests --enable_wasm_api_exception_catching --disable_rtti --build_dir ./build_wasm_inferencing_webgpu --use_webgpu --use_jsep --use_webnn diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index 2c953ca928..4accb413b6 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -209,6 +209,8 @@ else() target_link_libraries(onnxruntime_webassembly PRIVATE tensorboard) endif() + set(onnxruntime_webassembly_script_deps "${ONNXRUNTIME_ROOT}/wasm/pre.js") + if (onnxruntime_USE_JSEP) set(EXPORTED_FUNCTIONS "_malloc,_free,_JsepOutput,_JsepGetNodeName") else() @@ -312,11 +314,13 @@ else() target_link_options(onnxruntime_webassembly PRIVATE --post-js "${ONNXRUNTIME_ROOT}/wasm/js_post_js_64.js" ) + list(APPEND onnxruntime_webassembly_script_deps "${ONNXRUNTIME_ROOT}/wasm/js_post_js_64.js") else () set(MAXIMUM_MEMORY "4294967296") target_link_options(onnxruntime_webassembly PRIVATE --post-js "${ONNXRUNTIME_ROOT}/wasm/js_post_js.js" ) + list(APPEND onnxruntime_webassembly_script_deps "${ONNXRUNTIME_ROOT}/wasm/js_post_js.js") endif () target_link_options(onnxruntime_webassembly PRIVATE @@ -370,7 +374,6 @@ jsepDownload:_pp_") "SHELL:-s SIGNATURE_CONVERSIONS='${SIGNATURE_CONVERSIONS}'" ) endif () - set_target_properties(onnxruntime_webassembly PROPERTIES LINK_DEPENDS ${ONNXRUNTIME_ROOT}/wasm/pre.js) if (onnxruntime_USE_JSEP) # NOTE: "-s ASYNCIFY=1" is required for JSEP to work with WebGPU @@ -383,7 +386,7 @@ jsepDownload:_pp_") "SHELL:-s ASYNCIFY=1" "SHELL:-s ASYNCIFY_STACK_SIZE=65536" ) - set_target_properties(onnxruntime_webassembly PROPERTIES LINK_DEPENDS ${ONNXRUNTIME_ROOT}/wasm/pre-jsep.js) + list(APPEND onnxruntime_webassembly_script_deps "${ONNXRUNTIME_ROOT}/wasm/pre-jsep.js") if (onnxruntime_ENABLE_WEBASSEMBLY_MEMORY64) target_link_options(onnxruntime_webassembly PRIVATE @@ -397,6 +400,14 @@ jsepDownload:_pp_") target_compile_definitions(onnxruntime_webassembly PRIVATE USE_WEBGPU=1) endif() + if (onnxruntime_USE_JSEP OR onnxruntime_USE_WEBGPU OR onnxruntime_USE_WEBNN) + # if any of the above is enabled, we need to use the asyncify library + target_link_options(onnxruntime_webassembly PRIVATE + "SHELL:--pre-js \"${ONNXRUNTIME_ROOT}/wasm/pre-async.js\"" + ) + list(APPEND onnxruntime_webassembly_script_deps "${ONNXRUNTIME_ROOT}/wasm/pre-async.js") + endif() + if (onnxruntime_EMSCRIPTEN_SETTINGS) foreach(setting IN LISTS onnxruntime_EMSCRIPTEN_SETTINGS) target_link_options(onnxruntime_webassembly PRIVATE "SHELL:-s ${setting}") @@ -449,6 +460,8 @@ jsepDownload:_pp_") ) endif() + set_target_properties(onnxruntime_webassembly PROPERTIES LINK_DEPENDS "${onnxruntime_webassembly_script_deps}") + set(target_name_list ort) if (onnxruntime_ENABLE_TRAINING_APIS) @@ -522,9 +535,9 @@ jsepDownload:_pp_") add_custom_command( TARGET onnxruntime_webassembly POST_BUILD - COMMAND ${CMAKE_COMMAND} -E echo "Backup file at $.bak" + # Backup file at $.bak COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$.bak" - COMMAND ${CMAKE_COMMAND} -E echo "Performing workaround for $" + COMMAND ${CMAKE_COMMAND} -E echo "Performing post-process for $" COMMAND ${NODE_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/wasm_post_build.js" "$" ) endif() diff --git a/js/build_webgpu.bat b/js/build_webgpu.bat new file mode 100644 index 0000000000..95413509e7 --- /dev/null +++ b/js/build_webgpu.bat @@ -0,0 +1,79 @@ +@echo off + +rem build_webgpu.bat --- build onnxruntime-web with WebGPU EP +rem +rem Usage: +rem build_webgpu.bat config [clean] +rem +rem Options: +rem config Build configuration, "d" or "r" +rem clean Perform a clean build, "clean" or empty + +setlocal enabledelayedexpansion + +set ROOT=%~dp0..\ +set BUILD_DIR=%ROOT%build_webgpu + +:arg1 +if ["%~1"]==["d"] ( + set CONFIG=Debug + set CONFIG_EXTRA_FLAG= + @rem --enable_wasm_profiling --wasm_run_tests_in_browser + @rem --cmake_extra_defines onnxruntime_ENABLE_WEBASSEMBLY_OUTPUT_OPTIMIZED_MODEL=1 + @rem --enable_wasm_debug_info + goto :arg2 +) +if ["%~1"]==["r"] ( + set CONFIG=Release + set CONFIG_EXTRA_FLAG= + @rem --enable_wasm_api_exception_catching --disable_rtti + goto :arg2 +) +echo Invalid configuration "%~1", must be "d"(Debug) or "r"(Release) +exit /b 1 + +:arg2 +if ["%~2"]==["clean"] ( + goto :clean +) +if not exist "%ROOT%js\web\dist" ( + goto :npm_ci +) + +goto :build_wasm + +:clean +if exist "%BUILD_DIR%" ( + rd /s /q %BUILD_DIR% +) + +pushd %ROOT% +git submodule sync --recursive +git submodule update --init --recursive +popd + +:npm_ci +pushd %ROOT%js +call npm ci +popd +pushd %ROOT%js\common +call npm ci +popd +pushd %ROOT%js\web +call npm ci +call npm run pull:wasm +popd + +:build_wasm + +set PATH=C:\Program Files\Git\usr\bin;%PATH% + +call %ROOT%build.bat --config %CONFIG% %CONFIG_EXTRA_FLAG% --skip_submodule_sync --build_wasm --target onnxruntime_webassembly --skip_tests^ + --enable_wasm_simd --enable_wasm_threads --use_jsep --use_webnn --use_webgpu --build_dir %BUILD_DIR% + +IF NOT "%ERRORLEVEL%" == "0" ( + exit /b %ERRORLEVEL% +) + +copy /Y %BUILD_DIR%\%CONFIG%\ort-wasm-simd-threaded.jsep.wasm %ROOT%js\web\dist\ +copy /Y %BUILD_DIR%\%CONFIG%\ort-wasm-simd-threaded.jsep.mjs %ROOT%js\web\dist\ diff --git a/js/web/lib/build-def.d.ts b/js/web/lib/build-def.d.ts index 59f64a3179..83a52ebaef 100644 --- a/js/web/lib/build-def.d.ts +++ b/js/web/lib/build-def.d.ts @@ -40,6 +40,13 @@ interface BuildDefinitions { */ readonly ENABLE_BUNDLE_WASM_JS: boolean; + /** + * defines whether to use WebGPU EP instead of JSEP for WebGPU backend. + * + * This flag requires the corresponding WebAssembly artifact to be built with `--use_webgpu` flag. + */ + readonly USE_WEBGPU_EP: boolean; + // #endregion // #region Build definitions for ESM diff --git a/js/web/lib/wasm/ep-webgpu/README.md b/js/web/lib/wasm/ep-webgpu/README.md new file mode 100644 index 0000000000..e9e6313339 --- /dev/null +++ b/js/web/lib/wasm/ep-webgpu/README.md @@ -0,0 +1,11 @@ +This folder "ep-webgpu" contains required TypeScript implementation for WebGPU EP support. + +"ep-webgpu" here contains "ep" in the name, to distinguish it from other the WebGPU implementation in the JSEP folder: + +- WebGPU EP is a C++ implementation. It uses the WebGPU C/C++ API provided by Dawn in the code. Emscripten will compile + the C++ code to WebAssembly, and add internal JavaScript code to make it work in the browser. +- JSEP (JavaScript Execution Provider) is a hybrid implementation. It contains both JavaScript and C++ code, including + the interop between them. It uses the WebGPU JavaScript API provided by the browser. + +For WebGPU backend, when build definition `BUILD_DEFS.USE_WEBGPU_EP` is `true`, it is considered the WebGPU EP will be +used. Otherwise, the JSEP will be used. diff --git a/js/web/lib/wasm/session-options.ts b/js/web/lib/wasm/session-options.ts index 17e5642478..809f6d5fe6 100644 --- a/js/web/lib/wasm/session-options.ts +++ b/js/web/lib/wasm/session-options.ts @@ -54,11 +54,11 @@ const appendDefaultOptions = (options: InferenceSession.SessionOptions): void => } }; -const setExecutionProviders = ( +const setExecutionProviders = async ( sessionOptionsHandle: number, executionProviders: readonly InferenceSession.ExecutionProviderConfig[], allocs: number[], -): void => { +): Promise => { for (const ep of executionProviders) { let epName = typeof ep === 'string' ? ep : ep.name; @@ -80,17 +80,24 @@ const setExecutionProviders = ( } break; case 'webgpu': - epName = 'JS'; - if (typeof ep !== 'string') { - const webgpuOptions = ep as InferenceSession.WebGpuExecutionProviderOption; - if (webgpuOptions?.preferredLayout) { - if (webgpuOptions.preferredLayout !== 'NCHW' && webgpuOptions.preferredLayout !== 'NHWC') { - throw new Error(`preferredLayout must be either 'NCHW' or 'NHWC': ${webgpuOptions.preferredLayout}`); - } - const keyDataOffset = allocWasmString('preferredLayout', allocs); - const valueDataOffset = allocWasmString(webgpuOptions.preferredLayout, allocs); - if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !== 0) { - checkLastError(`Can't set a session config entry: 'preferredLayout' - ${webgpuOptions.preferredLayout}.`); + if (BUILD_DEFS.USE_WEBGPU_EP) { + epName = 'WebGPU'; + // TODO: session options + } else { + epName = 'JS'; + if (typeof ep !== 'string') { + const webgpuOptions = ep as InferenceSession.WebGpuExecutionProviderOption; + if (webgpuOptions?.preferredLayout) { + if (webgpuOptions.preferredLayout !== 'NCHW' && webgpuOptions.preferredLayout !== 'NHWC') { + throw new Error(`preferredLayout must be either 'NCHW' or 'NHWC': ${webgpuOptions.preferredLayout}`); + } + const keyDataOffset = allocWasmString('preferredLayout', allocs); + const valueDataOffset = allocWasmString(webgpuOptions.preferredLayout, allocs); + if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !== 0) { + checkLastError( + `Can't set a session config entry: 'preferredLayout' - ${webgpuOptions.preferredLayout}.`, + ); + } } } } @@ -103,13 +110,13 @@ const setExecutionProviders = ( } const epNameDataOffset = allocWasmString(epName, allocs); - if (getInstance()._OrtAppendExecutionProvider(sessionOptionsHandle, epNameDataOffset) !== 0) { + if ((await getInstance()._OrtAppendExecutionProvider(sessionOptionsHandle, epNameDataOffset, 0, 0, 0)) !== 0) { checkLastError(`Can't append execution provider: ${epName}.`); } } }; -export const setSessionOptions = (options?: InferenceSession.SessionOptions): [number, number[]] => { +export const setSessionOptions = async (options?: InferenceSession.SessionOptions): Promise<[number, number[]]> => { const wasm = getInstance(); let sessionOptionsHandle = 0; const allocs: number[] = []; @@ -155,7 +162,7 @@ export const setSessionOptions = (options?: InferenceSession.SessionOptions): [n } if (sessionOptions.executionProviders) { - setExecutionProviders(sessionOptionsHandle, sessionOptions.executionProviders, allocs); + await setExecutionProviders(sessionOptionsHandle, sessionOptions.executionProviders, allocs); } if (sessionOptions.enableGraphCapture !== undefined) { diff --git a/js/web/lib/wasm/wasm-core-impl.ts b/js/web/lib/wasm/wasm-core-impl.ts index da8939cd02..4f93dd4ec4 100644 --- a/js/web/lib/wasm/wasm-core-impl.ts +++ b/js/web/lib/wasm/wasm-core-impl.ts @@ -102,52 +102,82 @@ export const initRuntime = async (env: Env): Promise => { * @param epName */ export const initEp = async (env: Env, epName: string): Promise => { + // initialize ASYNCIFY support + getInstance().asyncInit?.(); + + let adapter: GPUAdapter | null = env.webgpu?.adapter; + if (epName === 'webgpu') { + // perform WebGPU availability check + if (typeof navigator === 'undefined' || !navigator.gpu) { + throw new Error('WebGPU is not supported in current environment'); + } + + if (!adapter) { + // if adapter is not set, request a new adapter. + const powerPreference = env.webgpu.powerPreference; + if (powerPreference !== undefined && powerPreference !== 'low-power' && powerPreference !== 'high-performance') { + throw new Error(`Invalid powerPreference setting: "${powerPreference}"`); + } + const forceFallbackAdapter = env.webgpu.forceFallbackAdapter; + if (forceFallbackAdapter !== undefined && typeof forceFallbackAdapter !== 'boolean') { + throw new Error(`Invalid forceFallbackAdapter setting: "${forceFallbackAdapter}"`); + } + adapter = await navigator.gpu.requestAdapter({ powerPreference, forceFallbackAdapter }); + if (!adapter) { + throw new Error( + 'Failed to get GPU adapter. ' + + 'You may need to enable flag "--enable-unsafe-webgpu" if you are using Chrome.', + ); + } + } else { + // if adapter is set, validate it. + if ( + typeof adapter.limits !== 'object' || + typeof adapter.features !== 'object' || + typeof adapter.requestDevice !== 'function' + ) { + throw new Error('Invalid GPU adapter set in `env.webgpu.adapter`. It must be a GPUAdapter object.'); + } + } + } + + if (BUILD_DEFS.USE_WEBGPU_EP) { + // const requiredFeatures: GPUFeatureName[] = []; + // const deviceDescriptor: GPUDeviceDescriptor = { + // requiredLimits: { + // maxComputeWorkgroupStorageSize: adapter.limits.maxComputeWorkgroupStorageSize, + // maxComputeWorkgroupsPerDimension: adapter.limits.maxComputeWorkgroupsPerDimension, + // maxStorageBufferBindingSize: adapter.limits.maxStorageBufferBindingSize, + // maxBufferSize: adapter.limits.maxBufferSize, + // maxComputeInvocationsPerWorkgroup: adapter.limits.maxComputeInvocationsPerWorkgroup, + // maxComputeWorkgroupSizeX: adapter.limits.maxComputeWorkgroupSizeX, + // maxComputeWorkgroupSizeY: adapter.limits.maxComputeWorkgroupSizeY, + // maxComputeWorkgroupSizeZ: adapter.limits.maxComputeWorkgroupSizeZ, + // }, + // requiredFeatures, + // }; + // // Try requiring WebGPU features + // const requireFeatureIfAvailable = (feature: GPUFeatureName) => + // adapter.features.has(feature) && requiredFeatures.push(feature) && true; + // // Try chromium-experimental-timestamp-query-inside-passes and fallback to timestamp-query + // if (!requireFeatureIfAvailable('chromium-experimental-timestamp-query-inside-passes' as GPUFeatureName)) { + // requireFeatureIfAvailable('timestamp-query'); + // } + // requireFeatureIfAvailable('shader-f16'); + // // Try subgroups + // if (requireFeatureIfAvailable('subgroups' as GPUFeatureName)) { + // // If subgroups feature is available, also try subgroups-f16 + // requireFeatureIfAvailable('subgroups-f16' as GPUFeatureName); + // } + // this.device = await adapter.requestDevice(deviceDescriptor); + } if (!BUILD_DEFS.DISABLE_JSEP) { // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires const initJsep = require('./jsep/init').init; if (epName === 'webgpu') { - // perform WebGPU availability check - if (typeof navigator === 'undefined' || !navigator.gpu) { - throw new Error('WebGPU is not supported in current environment'); - } - - let adapter = env.webgpu.adapter as GPUAdapter | null; - if (!adapter) { - // if adapter is not set, request a new adapter. - const powerPreference = env.webgpu.powerPreference; - if ( - powerPreference !== undefined && - powerPreference !== 'low-power' && - powerPreference !== 'high-performance' - ) { - throw new Error(`Invalid powerPreference setting: "${powerPreference}"`); - } - const forceFallbackAdapter = env.webgpu.forceFallbackAdapter; - if (forceFallbackAdapter !== undefined && typeof forceFallbackAdapter !== 'boolean') { - throw new Error(`Invalid forceFallbackAdapter setting: "${forceFallbackAdapter}"`); - } - adapter = await navigator.gpu.requestAdapter({ powerPreference, forceFallbackAdapter }); - if (!adapter) { - throw new Error( - 'Failed to get GPU adapter. ' + - 'You may need to enable flag "--enable-unsafe-webgpu" if you are using Chrome.', - ); - } - } else { - // if adapter is set, validate it. - if ( - typeof adapter.limits !== 'object' || - typeof adapter.features !== 'object' || - typeof adapter.requestDevice !== 'function' - ) { - throw new Error('Invalid GPU adapter set in `env.webgpu.adapter`. It must be a GPUAdapter object.'); - } - } - await initJsep('webgpu', getInstance(), env, adapter); - } - if (epName === 'webnn') { + } else if (epName === 'webnn') { // perform WebNN availability check if (typeof navigator === 'undefined' || !(navigator as unknown as { ml: unknown }).ml) { throw new Error('WebNN is not supported in current environment'); @@ -270,7 +300,7 @@ export const createSession = async ( const outputNamesUTF8Encoded = []; try { - [sessionOptionsHandle, allocs] = setSessionOptions(options); + [sessionOptionsHandle, allocs] = await setSessionOptions(options); if (options?.externalData && wasm.mountExternalData) { const loadingPromises = []; diff --git a/js/web/lib/wasm/wasm-types.ts b/js/web/lib/wasm/wasm-types.ts index ebeac5dc9e..0762f023e7 100644 --- a/js/web/lib/wasm/wasm-types.ts +++ b/js/web/lib/wasm/wasm-types.ts @@ -52,6 +52,12 @@ export declare namespace JSEP { */ unmountExternalData(): void; + /** + * This function patches the WebAssembly module to support Asyncify. This function should be called at least once + * before any ORT API is called. + */ + asyncInit?(): void; + /** * This is the entry of JSEP initialization. This function is called once when initializing ONNX Runtime per * backend. This function initializes Asyncify support. If name is 'webgpu', also initializes WebGPU backend and @@ -316,7 +322,13 @@ export interface OrtInferenceAPIs { logVerbosityLevel: number, optimizedModelFilePath: number, ): number; - _OrtAppendExecutionProvider(sessionOptionsHandle: number, name: number): number; + _OrtAppendExecutionProvider( + sessionOptionsHandle: number, + name: number, + providerOptionsKeys: number, + providerOptionsValues: number, + numKeys: number, + ): Promise; _OrtAddFreeDimensionOverride(sessionOptionsHandle: number, name: number, dim: number): number; _OrtAddSessionConfigEntry(sessionOptionsHandle: number, configKey: number, configValue: number): number; _OrtReleaseSessionOptions(sessionOptionsHandle: number): number; diff --git a/js/web/script/build.ts b/js/web/script/build.ts index 6006de62b4..fb2029e1ae 100644 --- a/js/web/script/build.ts +++ b/js/web/script/build.ts @@ -57,6 +57,7 @@ const DEFAULT_DEFINE = { 'BUILD_DEFS.DISABLE_WASM': 'false', 'BUILD_DEFS.DISABLE_WASM_PROXY': 'false', 'BUILD_DEFS.ENABLE_BUNDLE_WASM_JS': 'false', + 'BUILD_DEFS.USE_WEBGPU_EP': 'true', 'BUILD_DEFS.IS_ESM': 'false', 'BUILD_DEFS.ESM_IMPORT_META_URL': 'undefined', diff --git a/onnxruntime/wasm/api.cc b/onnxruntime/wasm/api.cc index 7adfc6a2b2..9e95e86ab2 100644 --- a/onnxruntime/wasm/api.cc +++ b/onnxruntime/wasm/api.cc @@ -164,8 +164,12 @@ OrtSessionOptions* OrtCreateSessionOptions(size_t graph_optimization_level, return UNREGISTER_AUTO_RELEASE(session_options); } -int OrtAppendExecutionProvider(ort_session_options_handle_t session_options, const char* name) { - return CHECK_STATUS(SessionOptionsAppendExecutionProvider, session_options, name, nullptr, nullptr, 0); +int OrtAppendExecutionProvider(ort_session_options_handle_t session_options, + const char* name, + const char* const* provider_options_keys, + const char* const* provider_options_values, + size_t num_keys) { + return CHECK_STATUS(SessionOptionsAppendExecutionProvider, session_options, name, provider_options_keys, provider_options_values, num_keys); } int OrtAddFreeDimensionOverride(ort_session_options_handle_t session_options, diff --git a/onnxruntime/wasm/api.h b/onnxruntime/wasm/api.h index f44c515d98..448109960e 100644 --- a/onnxruntime/wasm/api.h +++ b/onnxruntime/wasm/api.h @@ -85,7 +85,10 @@ ort_session_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSessionOptions(size_t * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtAppendExecutionProvider(ort_session_options_handle_t session_options, - const char* name); + const char* name, + const char* const* provider_options_keys, + const char* const* provider_options_values, + size_t num_keys); /** * add a free dimension override for one dimension of a session's input. diff --git a/onnxruntime/wasm/js_post_js.js b/onnxruntime/wasm/js_post_js.js index b77d82fbd7..be5a4d3c74 100644 --- a/onnxruntime/wasm/js_post_js.js +++ b/onnxruntime/wasm/js_post_js.js @@ -2,6 +2,6 @@ // Licensed under the MIT License. -'use strict'; +"use strict"; Module["PTR_SIZE"] = 4; diff --git a/onnxruntime/wasm/js_post_js_64.js b/onnxruntime/wasm/js_post_js_64.js index b140df927e..b16383b746 100644 --- a/onnxruntime/wasm/js_post_js_64.js +++ b/onnxruntime/wasm/js_post_js_64.js @@ -2,6 +2,6 @@ // Licensed under the MIT License. -'use strict'; +"use strict"; Module["PTR_SIZE"] = 8; diff --git a/onnxruntime/wasm/pre-async.js b/onnxruntime/wasm/pre-async.js new file mode 100644 index 0000000000..a1e66d854d --- /dev/null +++ b/onnxruntime/wasm/pre-async.js @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +"use strict"; + +// +// This file contains the pre-run code for the ORT WebAssembly module. The code in this file will be injected into the +// final module using Emscripten's `--pre-js` option. +// +// This file will only be used in build with flag `-s ASYNCIFY=1`. + +/** + * initialize for asyncify support. + */ +let initAsyncImpl = () => { + // This is a simplified version of cwrap() with options.async === true (-sASYNCIFY=1) + // It removes some overhead in cwarp() and ccall() that we don't need. + // + // Currently in ASYNCIFY build, we only use this for the following functions: + // - OrtCreateSession() + // - OrtRun() + // - OrtRunWithBinding() + // - OrtBindInput() + // + // Note: about parameters "getFunc" and "setFunc": + // - Emscripten has different behaviors for Debug and Release builds for generating exported function wrapper. + // + // - In Debug build, it will generate a wrapper function for each exported function. For example, it generates a + // wrapper for OrtRun() like this (minified): + // ``` + // var _OrtRun = Module["_OrtRun"] = createExportWrapper("OrtRun"); + // ``` + // + // - In Release build, it will generate a lazy loading wrapper for each exported function. For example, it generates + // a wrapper for OrtRun() like this (minified): + // ``` + // d._OrtRun = (a, b, c, e, f, h, l, q) => (d._OrtRun = J.ka)(a, b, c, e, f, h, l, q); + // ``` + // + // The behavior of these two wrappers are different. The debug build will assign `Module["_OrtRun"]` only once + // because `createExportWrapper()` does not reset `Module["_OrtRun"]` inside. The release build, however, will + // reset d._OrtRun to J.ka when the first time it is called. + // + // The difference is important because we need to design the async wrapper in a way that it can handle both cases. + // + // Now, let's look at how the async wrapper is designed to work for both cases: + // + // - Debug build: + // 1. When Web assembly is being loaded, `Module["_OrtRun"]` is assigned to `createExportWrapper("OrtRun")`. + // 2. When the first time `Module["initAsync"]` is called, `Module["_OrtRun"]` is re-assigned to a new async + // wrapper function. + // Value of `Module["_OrtRun"]` will not be changed again. + // + // - Release build: + // 1. When Web assembly is being loaded, `Module["_OrtRun"]` is assigned to a lazy loading wrapper function. + // 2. When the first time `Module["initAsync"]` is called, `Module["_OrtRun"]` is re-assigned to a new async + // wrapper function. + // 3. When the first time `Module["_OrtRun"]` is called, the async wrapper will be called. It will call into this + // function: + // ``` + // (a, b, c, e, f, h, l, q) => (d._OrtRun = J.ka)(a, b, c, e, f, h, l, q); + // ``` + // This function will assign d._OrtRun (ie. the minimized `Module["_OrtRun"]`) to the real function (J.ka). + // 4. Since d._OrtRun is re-assigned, we need to update the async wrapper to re-assign its stored + // function to the updated value (J.ka), and re-assign the value of `d._OrtRun` back to the async wrapper. + // Value of `Module["_OrtRun"]` will not be changed again. + // + // The value of `Module["_OrtRun"]` will need to be assigned for 2 times for debug build and 4 times for release + // build. + // + // This is why we need this `getFunc` and `setFunc` parameters. They are used to get the current value of an + // exported function and set the new value of an exported function. + // + const wrapAsync = (func, getFunc, setFunc) => { + return (...args) => { + // cache the async data before calling the function. + const previousAsync = Asyncify.currData; + + const previousFunc = getFunc?.(); + const ret = func(...args); + const newFunc = getFunc?.(); + if (previousFunc !== newFunc) { + // The exported function has been updated. + // Set the sync function reference to the new function. + func = newFunc; + // Set the exported function back to the async wrapper. + setFunc(previousFunc); + // Remove getFunc and setFunc. They are no longer needed. + setFunc = null; + getFunc = null; + } + + // If the async data has been changed, it means that the function started an async operation. + if (Asyncify.currData != previousAsync) { + // returns the promise + return Asyncify.whenDone(); + } + // the function is synchronous. returns the result. + return ret; + }; + }; + + // replace the original functions with asyncified versions + Module["_OrtAppendExecutionProvider"] = wrapAsync( + Module["_OrtAppendExecutionProvider"], + () => Module["_OrtAppendExecutionProvider"], + (v) => (Module["_OrtAppendExecutionProvider"] = v) + ); + Module["_OrtCreateSession"] = wrapAsync( + Module["_OrtCreateSession"], + () => Module["_OrtCreateSession"], + (v) => (Module["_OrtCreateSession"] = v) + ); + Module["_OrtRun"] = wrapAsync( + Module["_OrtRun"], + () => Module["_OrtRun"], + (v) => (Module["_OrtRun"] = v) + ); + Module["_OrtRunWithBinding"] = wrapAsync( + Module["_OrtRunWithBinding"], + () => Module["_OrtRunWithBinding"], + (v) => (Module["_OrtRunWithBinding"] = v) + ); + Module["_OrtBindInput"] = wrapAsync( + Module["_OrtBindInput"], + () => Module["_OrtBindInput"], + (v) => (Module["_OrtBindInput"] = v) + ); + + // If JSEP is enabled, wrap OrtRun() and OrtRunWithBinding() with asyncify. + if (typeof jsepRunAsync !== "undefined") { + Module["_OrtRun"] = jsepRunAsync(Module["_OrtRun"]); + Module["_OrtRunWithBinding"] = jsepRunAsync(Module["_OrtRunWithBinding"]); + } + + // remove this function to make sure it is called only once. + initAsyncImpl = undefined; +}; + +Module["asyncInit"] = () => { + initAsyncImpl?.(); +}; diff --git a/onnxruntime/wasm/pre-jsep.js b/onnxruntime/wasm/pre-jsep.js index 45e2475548..9ccb214a30 100644 --- a/onnxruntime/wasm/pre-jsep.js +++ b/onnxruntime/wasm/pre-jsep.js @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -'use strict'; +"use strict"; // // This file contains the pre-run code for the ORT WebAssembly module. The code in this file will be injected into the @@ -9,241 +9,143 @@ // // This file will only be used in build with flag `--use_jsep`. +// This is a wrapper for OrtRun() and OrtRunWithBinding() to ensure that Promises are handled correctly. +const jsepRunAsync = (runAsyncFunc) => { + return async (...args) => { + try { + // Module.jsepSessionState should be null, unless we are in the middle of a session. + // If it is not null, it means that the previous session has not finished yet. + if (Module.jsepSessionState) { + throw new Error("Session already started"); + } + const state = (Module.jsepSessionState = { + sessionHandle: args[0], + errors: [], + }); -/** - * initialize JSEP for asyncify support. - */ -let jsepInitAsync = () => { - // This is a simplified version of cwrap() with options.async === true (-sASYNCIFY=1) - // It removes some overhead in cwarp() and ccall() that we don't need. - // - // Currently in JSEP build, we only use this for the following functions: - // - OrtRun() - // - OrtRunWithBinding() - // - OrtBindInput() - // - // Note: about parameters "getFunc" and "setFunc": - // - Emscripten has different behaviors for Debug and Release builds for generating exported function wrapper. - // - // - In Debug build, it will generate a wrapper function for each exported function. For example, it generates a - // wrapper for OrtRun() like this (minified): - // ``` - // var _OrtRun = Module["_OrtRun"] = createExportWrapper("OrtRun"); - // ``` - // - // - In Release build, it will generate a lazy loading wrapper for each exported function. For example, it generates - // a wrapper for OrtRun() like this (minified): - // ``` - // d._OrtRun = (a, b, c, e, f, h, l, q) => (d._OrtRun = J.ka)(a, b, c, e, f, h, l, q); - // ``` - // - // The behavior of these two wrappers are different. The debug build will assign `Module["_OrtRun"]` only once - // because `createExportWrapper()` does not reset `Module["_OrtRun"]` inside. The release build, however, will - // reset d._OrtRun to J.ka when the first time it is called. - // - // The difference is important because we need to design the async wrapper in a way that it can handle both cases. - // - // Now, let's look at how the async wrapper is designed to work for both cases: - // - // - Debug build: - // 1. When Web assembly is being loaded, `Module["_OrtRun"]` is assigned to `createExportWrapper("OrtRun")`. - // 2. When the first time `Module["jsepInit"]` is called, `Module["_OrtRun"]` is re-assigned to a new async - // wrapper function. - // Value of `Module["_OrtRun"]` will not be changed again. - // - // - Release build: - // 1. When Web assembly is being loaded, `Module["_OrtRun"]` is assigned to a lazy loading wrapper function. - // 2. When the first time `Module["jsepInit"]` is called, `Module["_OrtRun"]` is re-assigned to a new async - // wrapper function. - // 3. When the first time `Module["_OrtRun"]` is called, the async wrapper will be called. It will call into this - // function: - // ``` - // (a, b, c, e, f, h, l, q) => (d._OrtRun = J.ka)(a, b, c, e, f, h, l, q); - // ``` - // This function will assign d._OrtRun (ie. the minimized `Module["_OrtRun"]`) to the real function (J.ka). - // 4. Since d._OrtRun is re-assigned, we need to update the async wrapper to re-assign its stored - // function to the updated value (J.ka), and re-assign the value of `d._OrtRun` back to the async wrapper. - // Value of `Module["_OrtRun"]` will not be changed again. - // - // The value of `Module["_OrtRun"]` will need to be assigned for 2 times for debug build and 4 times for release - // build. - // - // This is why we need this `getFunc` and `setFunc` parameters. They are used to get the current value of an - // exported function and set the new value of an exported function. - // - const jsepWrapAsync = (func, getFunc, setFunc) => { - return (...args) => { - // cache the async data before calling the function. - const previousAsync = Asyncify.currData; + // Run the acyncified function: OrtRun() or OrtRunWithBinding() + const ret = await runAsyncFunc(...args); - const previousFunc = getFunc?.(); - const ret = func(...args); - const newFunc = getFunc?.(); - if (previousFunc !== newFunc) { - // The exported function has been updated. - // Set the sync function reference to the new function. - func = newFunc; - // Set the exported function back to the async wrapper. - setFunc(previousFunc); - // Remove getFunc and setFunc. They are no longer needed. - setFunc = null; - getFunc = null; + // Check if the session is still valid. this object should be the same as the one we set above. + if (Module.jsepSessionState !== state) { + throw new Error("Session mismatch"); } - // If the async data has been changed, it means that the function started an async operation. - if (Asyncify.currData != previousAsync) { - // returns the promise - return Asyncify.whenDone(); + // Flush the backend. This will submit all pending commands to the GPU. + Module.jsepBackend?.["flush"](); + + // Await all pending promises. This includes GPU validation promises for diagnostic purposes. + const errorPromises = state.errors; + if (errorPromises.length > 0) { + let errors = await Promise.all(errorPromises); + errors = errors.filter((e) => e); + if (errors.length > 0) { + throw new Error(errors.join("\n")); + } } - // the function is synchronous. returns the result. + return ret; - }; + } finally { + Module.jsepSessionState = null; + } }; - - // This is a wrapper for OrtRun() and OrtRunWithBinding() to ensure that Promises are handled correctly. - const runAsync = (runAsyncFunc) => { - return async (...args) => { - try { - // Module.jsepSessionState should be null, unless we are in the middle of a session. - // If it is not null, it means that the previous session has not finished yet. - if (Module.jsepSessionState) { - throw new Error('Session already started'); - } - const state = Module.jsepSessionState = {sessionHandle: args[0], errors: []}; - - // Run the acyncified function: OrtRun() or OrtRunWithBinding() - const ret = await runAsyncFunc(...args); - - // Check if the session is still valid. this object should be the same as the one we set above. - if (Module.jsepSessionState !== state) { - throw new Error('Session mismatch'); - } - - // Flush the backend. This will submit all pending commands to the GPU. - Module.jsepBackend?.['flush'](); - - // Await all pending promises. This includes GPU validation promises for diagnostic purposes. - const errorPromises = state.errors; - if (errorPromises.length > 0) { - let errors = await Promise.all(errorPromises); - errors = errors.filter(e => e); - if (errors.length > 0) { - throw new Error(errors.join('\n')); - } - } - - return ret; - } finally { - Module.jsepSessionState = null; - } - }; - }; - - // replace the original functions with asyncified versions - Module['_OrtCreateSession'] = jsepWrapAsync( - Module['_OrtCreateSession'], - () => Module['_OrtCreateSession'], - v => Module['_OrtCreateSession'] = v); - Module['_OrtRun'] = runAsync(jsepWrapAsync( - Module['_OrtRun'], - () => Module['_OrtRun'], - v => Module['_OrtRun'] = v)); - Module['_OrtRunWithBinding'] = runAsync(jsepWrapAsync( - Module['_OrtRunWithBinding'], - () => Module['_OrtRunWithBinding'], - v => Module['_OrtRunWithBinding'] = v)); - Module['_OrtBindInput'] = jsepWrapAsync( - Module['_OrtBindInput'], - () => Module['_OrtBindInput'], - v => Module['_OrtBindInput'] = v); - - // remove this function to make sure it is called only once. - jsepInitAsync = undefined; }; - /** - * initialize JSEP for WebGPU. + * initialize JSEP for WebGPU and WebNN. */ -Module['jsepInit'] = (name, params) => { - jsepInitAsync?.(); - - if (name === 'webgpu') { - [Module.jsepBackend, - Module.jsepAlloc, - Module.jsepFree, - Module.jsepCopy, - Module.jsepCopyAsync, - Module.jsepCreateKernel, - Module.jsepReleaseKernel, - Module.jsepRunKernel, - Module.jsepCaptureBegin, - Module.jsepCaptureEnd, - Module.jsepReplay] = params; +Module["jsepInit"] = (name, params) => { + if (name === "webgpu") { + [ + Module.jsepBackend, + Module.jsepAlloc, + Module.jsepFree, + Module.jsepCopy, + Module.jsepCopyAsync, + Module.jsepCreateKernel, + Module.jsepReleaseKernel, + Module.jsepRunKernel, + Module.jsepCaptureBegin, + Module.jsepCaptureEnd, + Module.jsepReplay, + ] = params; // expose webgpu backend functions const backend = Module.jsepBackend; - Module['jsepRegisterBuffer'] = (sessionId, index, buffer, size) => { - return backend['registerBuffer'](sessionId, index, buffer, size); + Module["jsepRegisterBuffer"] = (sessionId, index, buffer, size) => { + return backend["registerBuffer"](sessionId, index, buffer, size); }; - Module['jsepGetBuffer'] = (dataId) => { - return backend['getBuffer'](dataId); + Module["jsepGetBuffer"] = (dataId) => { + return backend["getBuffer"](dataId); }; - Module['jsepCreateDownloader'] = (gpuBuffer, size, type) => { - return backend['createDownloader'](gpuBuffer, size, type); + Module["jsepCreateDownloader"] = (gpuBuffer, size, type) => { + return backend["createDownloader"](gpuBuffer, size, type); }; - Module['jsepOnCreateSession'] = sessionId => { - backend['onCreateSession'](sessionId); + Module["jsepOnCreateSession"] = (sessionId) => { + backend["onCreateSession"](sessionId); }; - Module['jsepOnReleaseSession'] = sessionId => { - backend['onReleaseSession'](sessionId); + Module["jsepOnReleaseSession"] = (sessionId) => { + backend["onReleaseSession"](sessionId); }; - Module['jsepOnRunStart'] = sessionId => { - return backend['onRunStart'](sessionId); + Module["jsepOnRunStart"] = (sessionId) => { + return backend["onRunStart"](sessionId); }; Module.jsepUploadExternalBuffer = (dataId, buffer) => { - backend['upload'](dataId, buffer); + backend["upload"](dataId, buffer); }; - } else if (name === 'webnn') { + } else if (name === "webnn") { // Functions called from EM_ASM need to be assigned in a way that can be minified. // Functions called via emscripten::val::module_property need to be assigned by name so that the minifier doesn't // change the name. - [Module.jsepBackend, - Module.jsepReserveTensorId, - Module.jsepReleaseTensorId, - Module['jsepEnsureTensor'], - Module.jsepUploadTensor, - Module['jsepDownloadTensor'], + [ + Module.jsepBackend, + Module.jsepReserveTensorId, + Module.jsepReleaseTensorId, + Module["jsepEnsureTensor"], + Module.jsepUploadTensor, + Module["jsepDownloadTensor"], ] = params; // This function is called from both JS and an EM_ASM block, it needs both a minifiable name and an explicit name. - Module['jsepReleaseTensorId'] = Module.jsepReleaseTensorId; + Module["jsepReleaseTensorId"] = Module.jsepReleaseTensorId; // Functions called from JS also need to have explicit names. const backend = Module.jsepBackend; - Module['jsepOnRunStart'] = sessionId => { - return backend['onRunStart'](sessionId); + Module["jsepOnRunStart"] = (sessionId) => { + return backend["onRunStart"](sessionId); }; - Module['jsepRegisterMLContext'] = (sessionId, mlContext) => { - backend['registerMLContext'](sessionId, mlContext); + Module["jsepRegisterMLContext"] = (sessionId, mlContext) => { + backend["registerMLContext"](sessionId, mlContext); }; - Module['jsepOnReleaseSession'] = sessionId => { - backend['onReleaseSession'](sessionId); + Module["jsepOnReleaseSession"] = (sessionId) => { + backend["onReleaseSession"](sessionId); }; - Module['jsepCreateMLTensorDownloader'] = (tensorId, type) => { - return backend['createMLTensorDownloader'](tensorId, type); - } - Module['jsepRegisterMLTensor'] = (tensor, dataType, shape) => { - return backend['registerMLTensor'](tensor, dataType, shape); + Module["jsepCreateMLTensorDownloader"] = (tensorId, type) => { + return backend["createMLTensorDownloader"](tensorId, type); }; - Module['jsepCreateMLContext'] = (optionsOrGpuDevice) => { - return backend['createMLContext'](optionsOrGpuDevice); + Module["jsepRegisterMLTensor"] = (tensor, dataType, shape) => { + return backend["registerMLTensor"](tensor, dataType, shape); }; - Module['jsepRegisterMLConstant'] = (externalFilePath, dataOffset, dataLength, builder, desc) => { - return backend['registerMLConstant']( - externalFilePath, dataOffset, dataLength, builder, desc, Module.MountedFiles); + Module["jsepCreateMLContext"] = (optionsOrGpuDevice) => { + return backend["createMLContext"](optionsOrGpuDevice); + }; + Module["jsepRegisterMLConstant"] = ( + externalFilePath, + dataOffset, + dataLength, + builder, + desc + ) => { + return backend["registerMLConstant"]( + externalFilePath, + dataOffset, + dataLength, + builder, + desc, + Module.MountedFiles + ); }; } }; diff --git a/onnxruntime/wasm/pre.js b/onnxruntime/wasm/pre.js index 9b5f3ce545..6da28fc355 100644 --- a/onnxruntime/wasm/pre.js +++ b/onnxruntime/wasm/pre.js @@ -1,21 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -'use strict'; +"use strict"; // // This file contains the pre-run code for the ORT WebAssembly module. The code in this file will be injected into the // final module using Emscripten's `--pre-js` option. - /** * Mount external data files of a model to an internal map, which will be used during session initialization. * * @param {string} externalDataFilesPath * @param {Uint8Array} externalDataFilesData */ -Module['mountExternalData'] = (externalDataFilePath, externalDataFileData) => { - if (externalDataFilePath.startsWith('./')) { +Module["mountExternalData"] = (externalDataFilePath, externalDataFileData) => { + if (externalDataFilePath.startsWith("./")) { externalDataFilePath = externalDataFilePath.substring(2); } const files = Module.MountedFiles || (Module.MountedFiles = new Map()); @@ -25,7 +24,7 @@ Module['mountExternalData'] = (externalDataFilePath, externalDataFileData) => { /** * Unmount external data files of a model. */ -Module['unmountExternalData'] = () => { +Module["unmountExternalData"] = () => { delete Module.MountedFiles; }; @@ -48,5 +47,7 @@ Module['unmountExternalData'] = () => { * * @suppress {checkVars} */ -var SharedArrayBuffer = globalThis.SharedArrayBuffer ?? - new WebAssembly.Memory({'initial': 0, 'maximum': 0, 'shared': true}).buffer.constructor; +var SharedArrayBuffer = + globalThis.SharedArrayBuffer ?? + new WebAssembly.Memory({ initial: 0, maximum: 0, shared: true }).buffer + .constructor; diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 811eb4b878..8bf146fc8b 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1360,9 +1360,6 @@ def generate_build_tree( raise BuildError("WebNN is only available for WebAssembly build.") cmake_args += ["-Donnxruntime_USE_WEBNN=ON"] - if args.use_jsep and args.use_webgpu: - raise BuildError("JSEP (--use_jsep) and WebGPU (--use_webgpu) cannot be enabled at the same time.") - if args.use_external_dawn and not args.use_webgpu: raise BuildError("External Dawn (--use_external_dawn) must be enabled with WebGPU (--use_webgpu).")