From bdefc6c4d8c6f8daa60433139a4e12a8f5746fb3 Mon Sep 17 00:00:00 2001 From: Yulong Wang Date: Fri, 7 May 2021 12:12:37 -0700 Subject: [PATCH] [js/web] support multi-thread for wasm backend (#7601) --- cmake/CMakeLists.txt | 12 ++- cmake/onnxruntime_webassembly.cmake | 2 +- js/.eslintrc.js | 8 +- js/.vscode/settings.json | 3 +- js/common/lib/backend-impl.ts | 2 +- js/common/lib/env.ts | 54 +++++++++- js/common/lib/inference-session.ts | 4 +- js/web/karma.conf.js | 3 +- js/web/lib/backend-onnxjs.ts | 26 +---- js/web/lib/backend-wasm.ts | 45 ++++---- js/web/lib/onnxjs/backends/backend-webgl.ts | 18 ++-- js/web/lib/onnxjs/instrument.ts | 1 - js/web/lib/wasm/binding/index.ts | 42 -------- .../lib/wasm/binding/ort-wasm-threaded.d.ts | 7 ++ js/web/lib/wasm/binding/ort-wasm.d.ts | 14 ++- js/web/lib/wasm/index.ts | 5 - js/web/lib/wasm/session-handler.ts | 6 +- js/web/lib/wasm/wasm-factory.ts | 102 ++++++++++++++++++ js/web/package.json | 5 +- js/web/script/build.ts | 48 ++++++--- js/web/script/test-runner-cli-args.ts | 26 +++-- js/web/script/test-runner-cli.ts | 36 ++++--- js/web/test/test-main.ts | 21 ++-- js/web/test/test-runner.ts | 19 +++- js/web/test/test-types.ts | 10 +- js/web/test/unittests/opset.ts | 1 - js/web/webpack.config.js | 66 +++++++----- onnxruntime/wasm/api.cc | 18 +++- onnxruntime/wasm/api.h | 4 +- 29 files changed, 380 insertions(+), 228 deletions(-) delete mode 100644 js/web/lib/wasm/binding/index.ts create mode 100644 js/web/lib/wasm/binding/ort-wasm-threaded.d.ts delete mode 100644 js/web/lib/wasm/index.ts create mode 100644 js/web/lib/wasm/wasm-factory.ts diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index c3d0a58303..97ff7b5119 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -270,11 +270,13 @@ if (NOT MSVC AND NOT onnxruntime_ENABLE_BITCODE) endif() if (onnxruntime_BUILD_WEBASSEMBLY) - # Enable LTO for release build - if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - # we don't set onnxruntime_ENABLE_LTO because it appends flag "-flto=thin" - # currently, wasm-ld does not work correctly with "-flto=thin" - # instead, we manually set CMAKE_CXX_FLAGS "-flto" + # Enable LTO for release single-thread build + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + # NOTES: + # (1) LTO does not work for WebAssembly multi-thread. (segment fault in worker) + # (2) "-flto=thin" does not work correctly for wasm-ld. + # we don't set onnxruntime_ENABLE_LTO because it appends flag "-flto=thin" + # instead, we manually set CMAKE_CXX_FLAGS "-flto" string(APPEND CMAKE_CXX_FLAGS " -flto") endif() diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index b4a1687fde..52d6926274 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -41,7 +41,7 @@ target_link_libraries(onnxruntime_webassembly PRIVATE set(EXTRA_EXPORTED_RUNTIME_METHODS "['stackAlloc','stackRestore','stackSave','UTF8ToString','stringToUTF8','lengthBytesUTF8']") -set_target_properties(onnxruntime_webassembly PROPERTIES LINK_FLAGS " \ +set_target_properties(onnxruntime_webassembly PROPERTIES LINK_FLAGS " \ -s \"EXTRA_EXPORTED_RUNTIME_METHODS=${EXTRA_EXPORTED_RUNTIME_METHODS}\" \ -s WASM=1 \ -s NO_EXIT_RUNTIME=0 \ diff --git a/js/.eslintrc.js b/js/.eslintrc.js index e134323f62..f955da5313 100644 --- a/js/.eslintrc.js +++ b/js/.eslintrc.js @@ -14,7 +14,9 @@ module.exports = { ], 2 ], 'import/no-extraneous-dependencies': ['error', { 'devDependencies': false }], - 'import/no-internal-modules': 'error', + 'import/no-internal-modules': ['error', { + 'allow': ['**/lib/**'], + }], 'import/no-unassigned-import': 'error', '@typescript-eslint/array-type': ['error', { 'default': 'array-simple' }], '@typescript-eslint/await-thenable': 'error', @@ -43,7 +45,7 @@ module.exports = { '@typescript-eslint/no-inferrable-types': 'error', '@typescript-eslint/no-misused-new': 'error', '@typescript-eslint/no-namespace': ['error', { 'allowDeclarations': true }], - '@typescript-eslint/no-non-null-assertion': 'error', + '@typescript-eslint/no-non-null-assertion': 'off', '@typescript-eslint/no-require-imports': 'error', '@typescript-eslint/no-unnecessary-type-assertion': 'error', '@typescript-eslint/no-unused-vars': ['error', { 'argsIgnorePattern': '^_' }], @@ -121,7 +123,6 @@ module.exports = { '@typescript-eslint/no-explicit-any': 'off', '@typescript-eslint/no-require-imports': 'off', '@typescript-eslint/no-var-requires': 'off', - '@typescript-eslint/no-non-null-assertion': 'off', '@typescript-eslint/no-unnecessary-type-assertion': 'off', 'camelcase': 'off', 'prefer-arrow/prefer-arrow-functions': 'off', @@ -141,7 +142,6 @@ module.exports = { // TODO: those rules are useful. should turn on them in future (webgl refactor) '@typescript-eslint/no-empty-function': 'off', '@typescript-eslint/explicit-module-boundary-types': 'off', - '@typescript-eslint/no-non-null-assertion': 'off', '@typescript-eslint/no-use-before-define': 'off', '@typescript-eslint/no-unnecessary-type-assertion': 'off', '@typescript-eslint/restrict-plus-operands': 'off', diff --git a/js/.vscode/settings.json b/js/.vscode/settings.json index 0795a5c586..3efa8770d3 100644 --- a/js/.vscode/settings.json +++ b/js/.vscode/settings.json @@ -36,5 +36,6 @@ "**/node_modules": true, "./types": true }, - "typescript.tsdk": "node_modules/typescript/lib" + "typescript.tsdk": "node_modules/typescript/lib", + "git.detectSubmodules": false } diff --git a/js/common/lib/backend-impl.ts b/js/common/lib/backend-impl.ts index f7246ba478..bdc11a2379 100644 --- a/js/common/lib/backend-impl.ts +++ b/js/common/lib/backend-impl.ts @@ -80,5 +80,5 @@ export const resolveBackend = async(backendHints: readonly string[]): Promise `[${e.name}] ${e.err}`).join(', ')}`); }; diff --git a/js/common/lib/env.ts b/js/common/lib/env.ts index 37489de451..8b09ed20ed 100644 --- a/js/common/lib/env.ts +++ b/js/common/lib/env.ts @@ -1,16 +1,66 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -interface Env { +export declare namespace Env { + export interface WebAssemblyFlags { + /** + * set or get number of thread(s). If omitted or set to 0, number of thread(s) will be determined by system. If set + * to 1, no worker thread will be spawned. + * + * This setting is available only when WebAssembly multithread feature is available in current context. + */ + numThreads?: number; + + /** + * Set or get a number specifying the timeout for initialization of WebAssembly backend, in milliseconds. A zero + * value indicates no timeout is set. (default is 0) + */ + initTimeout?: number; + } + + export interface WebGLFlags { + /** + * Set or get the WebGL Context ID (webgl or webgl2). Default value is webgl2. + */ + contextId?: 'webgl'|'webgl2'; + /** + * Set or get the maximum batch size for matmul. 0 means to disable batching. + */ + matmulMaxBatchSize?: number; + /** + * Set or get the texture cache mode. Default value is full. + */ + textureCacheMode?: 'initializerOnly'|'full'; + /** + * Set or get the packed texture mode + */ + pack?: boolean; + } +} + +export interface Env { /** * Indicate whether run in debug mode. */ debug?: boolean; + /** + * Represent a set of flags for WebAssembly + */ + wasm: Env.WebAssemblyFlags; + + /** + * Represent a set of flags for WebGL + */ + webgl: Env.WebGLFlags; + [name: string]: unknown; } /** * Represent a set of flags as a global singleton. */ -export const env: Env = {}; +export const env: Env = { + wasm: {}, + webgl: {} +}; diff --git a/js/common/lib/inference-session.ts b/js/common/lib/inference-session.ts index 8bfe18fba9..47bf9bc044 100644 --- a/js/common/lib/inference-session.ts +++ b/js/common/lib/inference-session.ts @@ -54,14 +54,14 @@ export declare namespace InferenceSession { /** * The intra OP threads number. * - * This setting is available only in ONNXRuntime (Node.js binding and react-native) or WebAssembly backend + * This setting is available only in ONNXRuntime (Node.js binding and react-native). */ intraOpNumThreads?: number; /** * The inter OP threads number. * - * This setting is available only in ONNXRuntime (Node.js binding and react-native) or WebAssembly backend + * This setting is available only in ONNXRuntime (Node.js binding and react-native). */ interOpNumThreads?: number; diff --git a/js/web/karma.conf.js b/js/web/karma.conf.js index eaa07256e8..5b8991093b 100644 --- a/js/web/karma.conf.js +++ b/js/web/karma.conf.js @@ -3,6 +3,7 @@ const bundleMode = require('minimist')(process.argv)['bundle-mode'] || 'dev'; // 'dev'|'perf'|undefined; const karmaPlugins = require('minimist')(process.argv)['karma-plugins'] || undefined; +const timeoutMocha = require('minimist')(process.argv)['timeout-mocha'] || 60000; const commonFile = bundleMode === 'dev' ? '../common/dist/ort-common.js' : '../common/dist/ort-common.min.js' const mainFile = bundleMode === 'dev' ? 'test/ort.dev.js' : 'test/ort.perf.js'; @@ -59,7 +60,7 @@ module.exports = function (config) { '/base/test/ort-wasm-threaded.worker.js': '/base/dist/ort-wasm-threaded.worker.js', }, plugins: karmaPlugins, - client: { captureConsole: true, mocha: { expose: ['body'], timeout: 60000 } }, + client: { captureConsole: true, mocha: { expose: ['body'], timeout: timeoutMocha } }, preprocessors: { mainFile: ['sourcemap'] }, reporters: ['mocha', 'BrowserStack'], browsers: [], diff --git a/js/web/lib/backend-onnxjs.ts b/js/web/lib/backend-onnxjs.ts index 7bbaeb7c7e..9a21ea0a40 100644 --- a/js/web/lib/backend-onnxjs.ts +++ b/js/web/lib/backend-onnxjs.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. /* eslint-disable import/no-internal-modules */ -import {Backend, env, InferenceSession, SessionHandler} from 'onnxruntime-common'; +import {Backend, InferenceSession, SessionHandler} from 'onnxruntime-common'; import {Session} from './onnxjs/session'; import {OnnxjsSessionHandler} from './onnxjs/session-handler'; @@ -30,27 +30,3 @@ class OnnxjsBackend implements Backend { } export const onnxjsBackend = new OnnxjsBackend(); - -export interface WebGLFlags { - /** - * set or get the WebGL Context ID (webgl or webgl2) - */ - contextId?: 'webgl'|'webgl2'; - /** - * set or get the maximum batch size for matmul. 0 means to disable batching. - */ - matmulMaxBatchSize?: number; - /** - * set or get the texture cache mode - */ - textureCacheMode?: 'initializerOnly'|'full'; - /** - * set or get the packed texture mode - */ - pack?: boolean; -} - -/** - * Represent a set of flags for ONNX.js backend. - */ -export const flags: WebGLFlags = env.webgl = env.webgl as WebGLFlags || {}; diff --git a/js/web/lib/backend-wasm.ts b/js/web/lib/backend-wasm.ts index 35cd12b828..bebaffb19b 100644 --- a/js/web/lib/backend-wasm.ts +++ b/js/web/lib/backend-wasm.ts @@ -3,11 +3,33 @@ import {Backend, env, InferenceSession, SessionHandler} from 'onnxruntime-common'; -import {init, OnnxruntimeWebAssemblySessionHandler} from './wasm'; +import {OnnxruntimeWebAssemblySessionHandler} from './wasm/session-handler'; +import {initializeWebAssembly} from './wasm/wasm-factory'; + +/** + * This function initializes all flags for WebAssembly. + * + * Those flags are accessible from `ort.env.wasm`. Users are allow to set those flags before the first inference session + * being created, to override default value. + */ +export const initializeFlags = (): void => { + if (typeof env.wasm.initTimeout !== 'number' || env.wasm.initTimeout < 0) { + env.wasm.initTimeout = 0; + } + + if (typeof env.wasm.numThreads !== 'number' || !Number.isInteger(env.wasm.numThreads) || env.wasm.numThreads < 0) { + env.wasm.numThreads = Math.ceil((navigator.hardwareConcurrency || 1) / 2); + } + env.wasm.numThreads = Math.min(4, env.wasm.numThreads); +}; class OnnxruntimeWebAssemblyBackend implements Backend { async init(): Promise { - await init(); + // populate wasm flags + initializeFlags(); + + // init wasm + await initializeWebAssembly(); } createSessionHandler(path: string, options?: InferenceSession.SessionOptions): Promise; createSessionHandler(buffer: Uint8Array, options?: InferenceSession.SessionOptions): Promise; @@ -29,22 +51,3 @@ class OnnxruntimeWebAssemblyBackend implements Backend { } export const wasmBackend = new OnnxruntimeWebAssemblyBackend(); - -export interface WebAssemblyFlags { - /** - * set or get number of worker(s) - * - * This setting is available only when WebAssembly multithread feature is available in current context. - */ - worker?: number; - - /** - * set or get a number specifying the timeout for initialization of WebAssembly backend, in milliseconds. - */ - initTimeout?: number; -} - -/** - * Represent a set of flags for WebAssembly backend. - */ -export const flags: WebAssemblyFlags = env.wasm = env.wasm as WebAssemblyFlags || {}; diff --git a/js/web/lib/onnxjs/backends/backend-webgl.ts b/js/web/lib/onnxjs/backends/backend-webgl.ts index 1b6900dc42..33b149eb98 100644 --- a/js/web/lib/onnxjs/backends/backend-webgl.ts +++ b/js/web/lib/onnxjs/backends/backend-webgl.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import {flags} from '../../backend-onnxjs'; +import {env} from 'onnxruntime-common'; import {Backend, SessionHandler} from '../backend'; import {Logger} from '../instrument'; import {Session} from '../session'; @@ -19,31 +19,31 @@ export class WebGLBackend implements Backend { glContext: WebGLContext; get contextId(): 'webgl'|'webgl2'|undefined { - return flags.contextId; + return env.webgl.contextId; } set contextId(value: 'webgl'|'webgl2'|undefined) { - flags.contextId = value; + env.webgl.contextId = value; } get matmulMaxBatchSize(): number|undefined { - return flags.matmulMaxBatchSize; + return env.webgl.matmulMaxBatchSize; } set matmulMaxBatchSize(value: number|undefined) { - flags.matmulMaxBatchSize = value; + env.webgl.matmulMaxBatchSize = value; } get textureCacheMode(): 'initializerOnly'|'full'|undefined { - return flags.textureCacheMode; + return env.webgl.textureCacheMode; } set textureCacheMode(value: 'initializerOnly'|'full'|undefined) { - flags.textureCacheMode = value; + env.webgl.textureCacheMode = value; } get pack(): boolean|undefined { - return flags.pack; + return env.webgl.pack; } set pack(value: boolean|undefined) { - flags.pack = value; + env.webgl.pack = value; } initialize(): boolean { diff --git a/js/web/lib/onnxjs/instrument.ts b/js/web/lib/onnxjs/instrument.ts index 6d64df0422..1be1de6764 100644 --- a/js/web/lib/onnxjs/instrument.ts +++ b/js/web/lib/onnxjs/instrument.ts @@ -202,7 +202,6 @@ namespace log { export function reset(config?: Logger.Config): void { LOGGER_CONFIG_MAP = {}; - // tslint:disable-next-line:no-backbone-get-set-outside-model set('', config || {}); } export function set(category: string, config: Logger.Config): void { diff --git a/js/web/lib/wasm/binding/index.ts b/js/web/lib/wasm/binding/index.ts deleted file mode 100644 index d147e10c80..0000000000 --- a/js/web/lib/wasm/binding/index.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import wasmModuleFactory, {BackendWasmModule} from './ort-wasm'; - -// some global parameters to deal with wasm binding initialization -let wasm: BackendWasmModule; -let initialized = false; -let initializing = false; - -/** - * initialize the WASM instance. - * - * this function should be called before any other calls to the WASM binding. - */ -export const init = async(): Promise => { - if (initialized) { - return Promise.resolve(); - } - if (initializing) { - throw new Error('multiple calls to \'init()\' detected.'); - } - - initializing = true; - - return new Promise((resolve, reject) => { - wasmModuleFactory().then( - initializedModule => { - // resolve init() promise - wasm = initializedModule; - initializing = false; - initialized = true; - resolve(); - }, - err => { - initializing = false; - reject(err); - }); - }); -}; - -export const getInstance = (): BackendWasmModule => wasm; diff --git a/js/web/lib/wasm/binding/ort-wasm-threaded.d.ts b/js/web/lib/wasm/binding/ort-wasm-threaded.d.ts new file mode 100644 index 0000000000..ad8e5f6636 --- /dev/null +++ b/js/web/lib/wasm/binding/ort-wasm-threaded.d.ts @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import {OrtWasmModule} from './ort-wasm'; + +declare const moduleFactory: EmscriptenModuleFactory; +export default moduleFactory; diff --git a/js/web/lib/wasm/binding/ort-wasm.d.ts b/js/web/lib/wasm/binding/ort-wasm.d.ts index ab68d51c2f..2c12322c84 100644 --- a/js/web/lib/wasm/binding/ort-wasm.d.ts +++ b/js/web/lib/wasm/binding/ort-wasm.d.ts @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -export interface BackendWasmModule extends EmscriptenModule { +export interface OrtWasmModule extends EmscriptenModule { + //#region emscripten functions stackSave(): number; stackRestore(stack: number): void; stackAlloc(size: number): number; @@ -9,8 +10,10 @@ export interface BackendWasmModule extends EmscriptenModule { UTF8ToString(offset: number): string; lengthBytesUTF8(str: string): number; stringToUTF8(str: string, offset: number, maxBytes: number): void; + //#endregion - _OrtInit(): void; + //#region ORT APIs + _OrtInit(numThreads: number, loggingLevel: number): void; _OrtCreateSession(dataOffset: number, dataLength: number): number; _OrtReleaseSession(sessionHandle: number): void; @@ -29,7 +32,12 @@ export interface BackendWasmModule extends EmscriptenModule { _OrtRun( sessionHandle: number, inputNamesOffset: number, inputsOffset: number, inputCount: number, outputNamesOffset: number, outputCount: number, outputsOffset: number): number; + //#endregion + + //#region config + mainScriptUrlOrBlob?: string|Blob; + //#endregion } -declare const moduleFactory: EmscriptenModuleFactory; +declare const moduleFactory: EmscriptenModuleFactory; export default moduleFactory; diff --git a/js/web/lib/wasm/index.ts b/js/web/lib/wasm/index.ts deleted file mode 100644 index dc4e1ca524..0000000000 --- a/js/web/lib/wasm/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -export {init} from './binding'; -export {OnnxruntimeWebAssemblySessionHandler} from './session-handler'; diff --git a/js/web/lib/wasm/session-handler.ts b/js/web/lib/wasm/session-handler.ts index ccd1f6cfb9..d0edc73f2d 100644 --- a/js/web/lib/wasm/session-handler.ts +++ b/js/web/lib/wasm/session-handler.ts @@ -2,8 +2,8 @@ // Licensed under the MIT License. import {onnx} from 'onnx-proto'; -import {InferenceSession, SessionHandler, Tensor, TypedTensor} from 'onnxruntime-common'; -import {getInstance} from './binding'; +import {env, InferenceSession, SessionHandler, Tensor, TypedTensor} from 'onnxruntime-common'; +import {getInstance} from './wasm-factory'; let ortInit: boolean; @@ -113,7 +113,7 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { loadModel(model: Uint8Array): void { const wasm = getInstance(); if (!ortInit) { - wasm._OrtInit(); + wasm._OrtInit(env.wasm.numThreads!, 2 /* LoggingLevel::Warning */); ortInit = true; } diff --git a/js/web/lib/wasm/wasm-factory.ts b/js/web/lib/wasm/wasm-factory.ts new file mode 100644 index 0000000000..c7bef35eaf --- /dev/null +++ b/js/web/lib/wasm/wasm-factory.ts @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import {env} from 'onnxruntime-common'; +import {OrtWasmModule} from './binding/ort-wasm'; +import ortWasmFactoryThreaded from './binding/ort-wasm-threaded.js'; +import ortWasmFactory from './binding/ort-wasm.js'; + +let wasm: OrtWasmModule; +let initialized = false; +let initializing = false; +let aborted = false; + +const isMultiThreadSupported = (): boolean => { + try { + // Test for transferability of SABs (needed for Firefox) + // https://groups.google.com/forum/#!msg/mozilla.dev.platform/IHkBZlHETpA/dwsMNchWEQAJ + new MessageChannel().port1.postMessage(new SharedArrayBuffer(1)); + // This typed array is a WebAssembly program containing threaded + // instructions. + return WebAssembly.validate(new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 4, 1, 96, 0, 0, 3, 2, 1, 0, 5, + 4, 1, 3, 1, 1, 10, 11, 1, 9, 0, 65, 0, 254, 16, 2, 0, 26, 11 + ])); + } catch (e) { + return false; + } +}; + +export const initializeWebAssembly = async(): Promise => { + if (initialized) { + return Promise.resolve(); + } + if (initializing) { + throw new Error('multiple calls to \'initializeWebAssembly()\' detected.'); + } + if (aborted) { + throw new Error('previous call to \'initializeWebAssembly()\' failed.'); + } + + initializing = true; + + // wasm flags are already initialized + const timeout = env.wasm.initTimeout!; + const numThreads = env.wasm.numThreads!; + + const useThreads = numThreads > 1 && isMultiThreadSupported(); + let isTimeout = false; + + const tasks: Array> = []; + + // promise for timeout + if (timeout > 0) { + tasks.push(new Promise((resolve) => { + setTimeout(() => { + isTimeout = true; + resolve(); + }, timeout); + })); + } + + // promise for module initialization + tasks.push(new Promise((resolve, reject) => { + const factory = useThreads ? ortWasmFactoryThreaded : ortWasmFactory; + const config: Partial = {}; + + if (useThreads) { + config.mainScriptUrlOrBlob = new Blob( + [`var ortWasmThreaded=(function(){var _scriptDir;return ${ortWasmFactoryThreaded.toString()}})();`], + {type: 'text/javascript'}); + } + + factory(config).then( + // wasm module initialized successfully + module => { + initializing = false; + initialized = true; + wasm = module; + resolve(); + }, + // wasm module failed to initialize + (what) => { + initializing = false; + aborted = true; + reject(what); + }); + })); + + await Promise.race(tasks); + + if (isTimeout) { + throw new Error(`WebAssembly backend initializing failed due to timeout: ${timeout}ms`); + } +}; + +export const getInstance = (): OrtWasmModule => { + if (initialized) { + return wasm; + } + + throw new Error('WebAssembly is not initialized yet.'); +}; diff --git a/js/web/package.json b/js/web/package.json index d5e83803b7..615981525e 100644 --- a/js/web/package.json +++ b/js/web/package.json @@ -75,6 +75,9 @@ "browser": { "fs": false, "path": false, - "util": false + "util": false, + "worker_threads": false, + "perf_hooks": false, + "os": false } } diff --git a/js/web/script/build.ts b/js/web/script/build.ts index 6cbdba480f..a85ce1a681 100644 --- a/js/web/script/build.ts +++ b/js/web/script/build.ts @@ -9,29 +9,51 @@ import * as path from 'path'; // CMD args const args = minimist(process.argv); -const MODE = args.config || 'prod'; // prod|dev|test -if (['prod', 'dev', 'test'].indexOf(MODE) === -1) { + +// --bundle-mode=prod (default) +// --bundle-mode=dev +// --bundle-mode=perf +const MODE = args['bundle-mode'] || 'prod'; +if (['prod', 'dev', 'perf'].indexOf(MODE) === -1) { throw new Error(`unknown build mode: ${MODE}`); } +// --wasm (default) +// --no-wasm +const WASM = typeof args.wasm === 'undefined' ? true : !!args.wasm; + // Path variables const WASM_BINDING_FOLDER = path.join(__dirname, '..', 'lib', 'wasm', 'binding'); const WASM_JS_PATH = path.join(WASM_BINDING_FOLDER, 'ort-wasm.js'); +const WASM_THREADED_JS_PATH = path.join(WASM_BINDING_FOLDER, 'ort-wasm-threaded.js'); const WASM_DIST_FOLDER = path.join(__dirname, '..', 'dist'); -const WASM_DIST_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm.wasm'); +const WASM_WASM_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm.wasm'); +const WASM_THREADED_WASM_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm-threaded.wasm'); +const WASM_THREADED_WORKER_JS_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm-threaded.worker.js'); -try { - npmlog.info('Build', `Ensure file: ${WASM_JS_PATH}`); - if (!fs.pathExistsSync(WASM_JS_PATH)) { - throw new Error(`file does not exist: ${WASM_JS_PATH}`); +function validateFile(path: string): void { + npmlog.info('Build', `Ensure file: ${path}`); + if (!fs.pathExistsSync(path)) { + throw new Error(`file does not exist: ${path}`); } - npmlog.info('Build', `Ensure file: ${WASM_DIST_PATH}`); - if (!fs.pathExistsSync(WASM_DIST_PATH)) { - throw new Error(`file does not exist: ${WASM_DIST_PATH}`); + if (fs.statSync(path).size === 0) { + throw new Error(`file is empty: ${path}`); } -} catch (e) { - npmlog.error('Build', `WebAssembly files are not ready. build WASM first. ERR: ${e}`); - throw e; +} + +if (WASM) { + npmlog.info('Build', 'Validating WebAssembly artifacts...'); + try { + validateFile(WASM_JS_PATH); + validateFile(WASM_THREADED_JS_PATH); + validateFile(WASM_WASM_PATH); + validateFile(WASM_THREADED_WASM_PATH); + validateFile(WASM_THREADED_WORKER_JS_PATH); + } catch (e) { + npmlog.error('Build', `WebAssembly files are not ready. build WASM first. ERR: ${e}`); + throw e; + } + npmlog.info('Build', 'Validating WebAssembly artifacts... DONE'); } npmlog.info('Build', 'Building bundle...'); diff --git a/js/web/script/test-runner-cli-args.ts b/js/web/script/test-runner-cli-args.ts index 9491e4b882..b2519e8617 100644 --- a/js/web/script/test-runner-cli-args.ts +++ b/js/web/script/test-runner-cli-args.ts @@ -3,10 +3,8 @@ import minimist from 'minimist'; import npmlog from 'npmlog'; -import {InferenceSession} from 'onnxruntime-common'; +import {Env, InferenceSession} from 'onnxruntime-common'; -import {WebGLFlags} from '../lib/backend-onnxjs'; -import {WebAssemblyFlags} from '../lib/backend-wasm'; import {Logger} from '../lib/onnxjs/instrument'; import {Test} from '../test/test-types'; @@ -14,7 +12,7 @@ import {Test} from '../test/test-types'; const HELP_MESSAGE = ` test-runner-cli -Run ONNX.js tests, models, benchmarks in different environments. +Run ONNX Runtime Web tests, models, benchmarks in different environments. Usage: test-runner-cli ... [options] @@ -31,7 +29,7 @@ Options: -h, --help Print this message. -d, --debug Specify to run test runner in debug mode. - Debug mode outputs verbose log for test runner, sets up ONNX.js environment debug flag, and keeps karma not to exit after tests completed. + Debug mode outputs verbose log for test runner, sets up environment debug flag, and keeps karma not to exit after tests completed. -b=<...>, --backend=<...> Specify one or more backend(s) to run the test upon. Backends can be one or more of the following, splitted by comma: webgl @@ -60,7 +58,7 @@ Options: *** Backend Options *** - --wasm-worker Set the WebAssembly worker number + --wasm-number-threads Set the WebAssembly number of threads --wasm-init-timeout Set the timeout for WebAssembly backend initialization, in milliseconds --webgl-context-id Set the WebGL context ID (webgl/webgl2) --webgl-matmul-max-batch-size Set the WebGL matmulMaxBatchSize @@ -151,9 +149,9 @@ export interface TestRunnerCliArgs { cudaOptions?: InferenceSession.CudaExecutionProviderOption; cudaFlags?: Record; wasmOptions?: InferenceSession.WebAssemblyExecutionProviderOption; - wasmFlags?: WebAssemblyFlags; + wasmFlags?: Env.WebAssemblyFlags; webglOptions?: InferenceSession.WebGLExecutionProviderOption; - webglFlags?: WebGLFlags; + webglFlags?: Env.WebGLFlags; noSandbox?: boolean; } @@ -246,23 +244,23 @@ function parseWasmOptions(_args: minimist.ParsedArgs): InferenceSession.WebAssem return {name: 'wasm'}; } -function parseWasmFlags(args: minimist.ParsedArgs): WebAssemblyFlags { - const worker = args['wasm-worker']; - if (typeof worker !== 'undefined' && typeof worker !== 'number') { - throw new Error('Flag "wasm-worker" must be a number value'); +function parseWasmFlags(args: minimist.ParsedArgs): Env.WebAssemblyFlags { + const numThreads = args['wasm-number-threads']; + if (typeof numThreads !== 'undefined' && typeof numThreads !== 'number') { + throw new Error('Flag "wasm-number-threads" must be a number value'); } const initTimeout = args['wasm-init-timeout']; if (typeof initTimeout !== 'undefined' && typeof initTimeout !== 'number') { throw new Error('Flag "wasm-init-timeout" must be a number value'); } - return {worker, initTimeout}; + return {numThreads, initTimeout}; } function parseWebglOptions(_args: minimist.ParsedArgs): InferenceSession.WebGLExecutionProviderOption { return {name: 'webgl'}; } -function parseWebglFlags(args: minimist.ParsedArgs): WebGLFlags { +function parseWebglFlags(args: minimist.ParsedArgs): Env.WebGLFlags { const contextId = args['webgl-context-id']; if (contextId !== undefined && contextId !== 'webgl' && contextId !== 'webgl2') { throw new Error('Flag "webgl-context-id" is invalid'); diff --git a/js/web/script/test-runner-cli.ts b/js/web/script/test-runner-cli.ts index a23c6c16bf..350747d130 100644 --- a/js/web/script/test-runner-cli.ts +++ b/js/web/script/test-runner-cli.ts @@ -410,7 +410,7 @@ function run(config: Test.Config) { npmlog.info('TestRunnerCli.Run', `(3/5) Retrieving npm bin folder... DONE, folder: ${npmBin}`); if (args.env === 'node') { - // STEP 4. use tsc to build ONNX.js + // STEP 4. use tsc to build ONNX Runtime Web npmlog.info('TestRunnerCli.Run', '(4/5) Running tsc...'); const tscCommand = path.join(npmBin, 'tsc'); const tsc = spawnSync(tscCommand, {shell: true, stdio: 'inherit'}); @@ -433,17 +433,20 @@ function run(config: Test.Config) { npmlog.info('TestRunnerCli.Run', '(5/5) Running mocha... DONE'); } else { - // STEP 4. use webpack to generate ONNX.js - npmlog.info('TestRunnerCli.Run', '(4/5) Running webpack to generate ONNX.js...'); - const webpackCommand = path.join(npmBin, 'webpack'); - const webpackArgs = ['--env', `--bundle-mode=${args.bundleMode}`]; - npmlog.info('TestRunnerCli.Run', `CMD: ${webpackCommand} ${webpackArgs.join(' ')}`); - const webpack = spawnSync(webpackCommand, webpackArgs, {shell: true, stdio: 'inherit'}); - if (webpack.status !== 0) { - console.error(webpack.error); - process.exit(webpack.status === null ? undefined : webpack.status); + // STEP 4. generate bundle + npmlog.info('TestRunnerCli.Run', '(4/5) Running build to generate bundle...'); + const buildCommand = `node ${path.join(__dirname, 'build')}`; + const buildArgs = [`--bundle-mode=${args.bundleMode}`]; + if (args.backends.indexOf('wasm') === -1) { + buildArgs.push('--no-wasm'); } - npmlog.info('TestRunnerCli.Run', '(4/5) Running webpack to generate ONNX.js... DONE'); + npmlog.info('TestRunnerCli.Run', `CMD: ${buildCommand} ${buildArgs.join(' ')}`); + const build = spawnSync(buildCommand, buildArgs, {shell: true, stdio: 'inherit'}); + if (build.status !== 0) { + console.error(build.error); + process.exit(build.status === null ? undefined : build.status); + } + npmlog.info('TestRunnerCli.Run', '(4/5) Running build to generate bundle... DONE'); // STEP 5. use Karma to run test npmlog.info('TestRunnerCli.Run', '(5/5) Running karma to start test runner...'); @@ -451,7 +454,7 @@ function run(config: Test.Config) { const browser = getBrowserNameFromEnv(args.env, args.debug); const karmaArgs = ['start', `--browsers ${browser}`]; if (args.debug) { - karmaArgs.push('--log-level info'); + karmaArgs.push('--log-level info --timeout-mocha 9999999'); } else { karmaArgs.push('--single-run'); } @@ -475,7 +478,6 @@ function run(config: Test.Config) { // check if we have the latest Edge installed: if (os.platform() === 'darwin' || (os.platform() === 'win32' && - // tslint:disable-next-line:no-require-imports no-submodule-imports require('@chiragrupani/karma-chromium-edge-launcher/dist/Utilities').default.GetEdgeExe('Edge') !== '')) { // use "@chiragrupani/karma-chromium-edge-launcher" karmaArgs.push( @@ -563,15 +565,15 @@ function saveConfig(config: Test.Config) { if (config.options.webglFlags && config.options.webglFlags.pack !== undefined) { setOptions += `ort.env.webgl.pack = ${JSON.stringify(config.options.webglFlags.pack)};`; } - if (config.options.wasmFlags && config.options.wasmFlags.worker !== undefined) { - setOptions += `ort.env.wasm.worker = ${JSON.stringify(config.options.wasmFlags.worker)};`; + if (config.options.wasmFlags && config.options.wasmFlags.numThreads !== undefined) { + setOptions += `ort.env.wasm.numThreads = ${JSON.stringify(config.options.wasmFlags.numThreads)};`; } if (config.options.wasmFlags && config.options.wasmFlags.initTimeout !== undefined) { setOptions += `ort.env.wasm.initTimeout = ${JSON.stringify(config.options.wasmFlags.initTimeout)};`; } // TODO: support onnxruntime nodejs binding - // if (config.model.some(testGroup => testGroup.tests.some(test => test.backend === 'onnxruntime'))) { - // setOptions += 'require(\'onnxjs-node\');'; + // if (config.model.some(testGroup => testGroup.tests.some(test => test.backend === 'cpu'))) { + // setOptions += 'require(\'onnxruntime-node\');'; // } fs.writeFileSync(path.join(TEST_ROOT, './testdata-config.js'), `${setOptions} diff --git a/js/web/test/test-main.ts b/js/web/test/test-main.ts index 1b7bc937dc..86b2dce371 100644 --- a/js/web/test/test-main.ts +++ b/js/web/test/test-main.ts @@ -10,10 +10,10 @@ import {Logger} from '../lib/onnxjs/instrument'; import {Test} from './test-types'; -const ONNX_JS_TEST_CONFIG = (ort.env as any).ORT_WEB_TEST_DATA as Test.Config; +const ORT_WEB_TEST_CONFIG = (ort.env as any).ORT_WEB_TEST_DATA as Test.Config; // Set logging configuration -for (const logConfig of ONNX_JS_TEST_CONFIG.log) { +for (const logConfig of ORT_WEB_TEST_CONFIG.log) { Logger.set(logConfig.category, logConfig.config); } @@ -21,15 +21,14 @@ import {ModelTestContext, OpTestContext, runModelTestSet, runOpTest} from './tes import {readJsonFile} from './test-shared'; // Unit test -if (ONNX_JS_TEST_CONFIG.unittest) { - // tslint:disable-next-line:no-require-imports +if (ORT_WEB_TEST_CONFIG.unittest) { require('./unittests'); } // Set file cache -if (ONNX_JS_TEST_CONFIG.fileCacheUrls) { +if (ORT_WEB_TEST_CONFIG.fileCacheUrls) { before('prepare file cache', async () => { - const allJsonCache = await Promise.all(ONNX_JS_TEST_CONFIG.fileCacheUrls!.map(readJsonFile)) as Test.FileCache[]; + const allJsonCache = await Promise.all(ORT_WEB_TEST_CONFIG.fileCacheUrls!.map(readJsonFile)) as Test.FileCache[]; for (const cache of allJsonCache) { ModelTestContext.setCache(cache); } @@ -52,7 +51,7 @@ function shouldSkipTest(test: Test.ModelTest|Test.OperatorTest) { } // ModelTests -for (const group of ONNX_JS_TEST_CONFIG.model) { +for (const group of ORT_WEB_TEST_CONFIG.model) { describe(`#ModelTest# - ${group.name}`, () => { for (const test of group.tests) { const describeTest = shouldSkipTest(test) ? describe.skip : describe; @@ -60,7 +59,7 @@ for (const group of ONNX_JS_TEST_CONFIG.model) { let context: ModelTestContext; before('prepare session', async () => { - context = await ModelTestContext.create(test, ONNX_JS_TEST_CONFIG.profile); + context = await ModelTestContext.create(test, ORT_WEB_TEST_CONFIG.profile); }); after('release session', () => { @@ -80,7 +79,7 @@ for (const group of ONNX_JS_TEST_CONFIG.model) { } // OpTests -for (const group of ONNX_JS_TEST_CONFIG.op) { +for (const group of ORT_WEB_TEST_CONFIG.op) { describe(`#OpTest# - ${group.name}`, () => { for (const test of group.tests) { const describeTest = shouldSkipTest(test) ? describe.skip : describe; @@ -90,13 +89,13 @@ for (const group of ONNX_JS_TEST_CONFIG.op) { before('Initialize Context', async () => { context = new OpTestContext(test); await context.init(); - if (ONNX_JS_TEST_CONFIG.profile) { + if (ORT_WEB_TEST_CONFIG.profile) { OpTestContext.profiler.start(); } }); after('Dispose Context', () => { - if (ONNX_JS_TEST_CONFIG.profile) { + if (ORT_WEB_TEST_CONFIG.profile) { OpTestContext.profiler.stop(); } context.dispose(); diff --git a/js/web/test/test-runner.ts b/js/web/test/test-runner.ts index 92a55ebc42..65251febcb 100644 --- a/js/web/test/test-runner.ts +++ b/js/web/test/test-runner.ts @@ -7,7 +7,6 @@ import {onnx as onnxProto} from 'onnx-proto'; import * as ort from 'onnxruntime-common'; import {extname} from 'path'; import {inspect, promisify} from 'util'; -import {flags as webglFlags} from '../lib/backend-onnxjs'; import {Attribute} from '../lib/onnxjs/attribute'; import {InferenceHandler, resolveBackend, SessionHandler} from '../lib/onnxjs/backend'; @@ -69,7 +68,9 @@ async function loadMlProto(_uriOrData: string|Uint8Array): Promise; wasmOptions?: InferenceSession.WebAssemblyExecutionProviderOption; - wasmFlags?: WebAssemblyFlags; + wasmFlags?: Env.WebAssemblyFlags; webglOptions?: InferenceSession.WebGLExecutionProviderOption; - webglFlags?: WebGLFlags; + webglFlags?: Env.WebGLFlags; } /** diff --git a/js/web/test/unittests/opset.ts b/js/web/test/unittests/opset.ts index c807f41894..e9f42aeb00 100644 --- a/js/web/test/unittests/opset.ts +++ b/js/web/test/unittests/opset.ts @@ -14,7 +14,6 @@ function createTestGraphNode(name: string, opType: string): Graph.Node { } function dummyOpConstructor(): Operator { - // tslint:disable-next-line:no-any return {} as any as Operator; } diff --git a/js/web/webpack.config.js b/js/web/webpack.config.js index 0851cee8cf..ad9ab94d6e 100644 --- a/js/web/webpack.config.js +++ b/js/web/webpack.config.js @@ -102,8 +102,14 @@ function buildTestRunnerConfig({ externals: { 'onnxruntime-common': 'ort', 'fs': 'fs', + 'perf_hooks': 'perf_hooks', + 'worker_threads': 'worker_threads', + }, + resolve: { + extensions: ['.ts', '.js'], + aliasFields: [], + fallback: { './binding/ort-wasm-threaded.js': false, './binding/ort-wasm.js': false } }, - resolve: { extensions: ['.ts', '.js'], aliasFields: [] }, plugins: [ new webpack.WatchIgnorePlugin({ paths: [/\.js$/, /\.d\.ts$/] }), new NodePolyfillPlugin() @@ -132,35 +138,39 @@ module.exports = () => { const bundleMode = args['bundle-mode'] || 'prod'; // 'prod'|'dev'|'perf'|undefined; const builds = []; - if (bundleMode === 'prod') { - builds.push( - // ort.min.js - buildOrtConfig({ suffix: '.min' }), - // ort.js - buildOrtConfig({ mode: 'development', devtool: 'inline-source-map' }), - // ort.es6.min.js - buildOrtConfig({ suffix: '.es6.min', target: 'es6' }), - // ort.es6.js - buildOrtConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }), + switch (bundleMode) { + case 'prod': + builds.push( + // ort.min.js + buildOrtConfig({ suffix: '.min' }), + // ort.js + buildOrtConfig({ mode: 'development', devtool: 'inline-source-map' }), + // ort.es6.min.js + buildOrtConfig({ suffix: '.es6.min', target: 'es6' }), + // ort.es6.js + buildOrtConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }), - // ort-web.min.js - buildOrtWebConfig({ suffix: '.min' }), - // ort-web.js - buildOrtWebConfig({ mode: 'development', devtool: 'inline-source-map' }), - // ort-web.es6.min.js - buildOrtWebConfig({ suffix: '.es6.min', target: 'es6' }), - // ort-web.es6.js - buildOrtWebConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }), + // ort-web.min.js + buildOrtWebConfig({ suffix: '.min' }), + // ort-web.js + buildOrtWebConfig({ mode: 'development', devtool: 'inline-source-map' }), + // ort-web.es6.min.js + buildOrtWebConfig({ suffix: '.es6.min', target: 'es6' }), + // ort-web.es6.js + buildOrtWebConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }), - // ort-web.node.js - buildOrtWebConfig({ suffix: '.node', format: 'commonjs' }), - ); - } - - if (bundleMode === 'dev') { - builds.push(buildTestRunnerConfig({ suffix: '.dev', mode: 'development', devtool: 'inline-source-map' })); - } else if (bundleMode === 'perf') { - builds.push(buildTestRunnerConfig({ suffix: '.perf', devtool: undefined })); + // ort-web.node.js + buildOrtWebConfig({ suffix: '.node', format: 'commonjs' }), + ); + break; + case 'dev': + builds.push(buildTestRunnerConfig({ suffix: '.dev', mode: 'development', devtool: 'inline-source-map' })); + break; + case 'perf': + builds.push(buildTestRunnerConfig({ suffix: '.perf' })); // TODO: .js.LICENSE.txt + break; + default: + throw new Error(`unsupported bundle mode: ${bundleMode}`); } return builds; diff --git a/onnxruntime/wasm/api.cc b/onnxruntime/wasm/api.cc index 523a2fc6fe..ccf94fd0d6 100644 --- a/onnxruntime/wasm/api.cc +++ b/onnxruntime/wasm/api.cc @@ -12,17 +12,25 @@ namespace { Ort::Env* g_env; } // namespace -void OrtInit() { - // TODO: allow user to specify logging - OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING; - g_env = new Ort::Env{logging_level, "Default"}; +void OrtInit(int numThreads, int logging_level) { +#if defined(__EMSCRIPTEN_PTHREADS__) + OrtThreadingOptions* tp_options; + Ort::ThrowOnError(Ort::GetApi().CreateThreadingOptions(&tp_options)); + Ort::ThrowOnError(Ort::GetApi().SetGlobalIntraOpNumThreads(tp_options, numThreads)); + Ort::ThrowOnError(Ort::GetApi().SetGlobalInterOpNumThreads(tp_options, 1)); + + g_env = new Ort::Env{tp_options, static_cast(logging_level), "Default"}; +#endif + g_env = new Ort::Env{static_cast(logging_level), "Default"}; } Ort::Session* OrtCreateSession(void* data, size_t data_length) { Ort::SessionOptions session_options; session_options.SetLogId("onnxruntime"); -#if !defined(__EMSCRIPTEN_PTHREADS__) +#if defined(__EMSCRIPTEN_PTHREADS__) + session_options.DisablePerSessionThreads(); +#else // must disable thread pool when WebAssembly multi-threads support is disabled. session_options.SetIntraOpNumThreads(1); #endif diff --git a/onnxruntime/wasm/api.h b/onnxruntime/wasm/api.h index 4f98a198a6..80e18b2dc2 100644 --- a/onnxruntime/wasm/api.h +++ b/onnxruntime/wasm/api.h @@ -24,8 +24,10 @@ extern "C" { /** * perform global initialization. should be called only once. + * @param numThreads number of total threads to use. + * @param logging_level default logging level. */ -void EMSCRIPTEN_KEEPALIVE OrtInit(); +void EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level); /** * create an instance of ORT session.