saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-07 04:39:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[js/web] support multi-thread for wasm backend (#7601)

Browse source
This commit is contained in:
Yulong Wang 2021-05-07 12:12:37 -07:00 committed by GitHub
parent 8ab0deceed
commit bdefc6c4d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 380 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
cmake/CMakeLists.txt
View file

@ -270,11 +270,13 @@ if (NOT MSVC AND NOT onnxruntime_ENABLE_BITCODE)
endif()
if (onnxruntime_BUILD_WEBASSEMBLY)
# Enable LTO for release build
if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
# we don't set onnxruntime_ENABLE_LTO because it appends flag "-flto=thin"
# currently, wasm-ld does not work correctly with "-flto=thin"
# instead, we manually set CMAKE_CXX_FLAGS "-flto"
# Enable LTO for release single-thread build
if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
# NOTES:
# (1) LTO does not work for WebAssembly multi-thread. (segment fault in worker)
# (2) "-flto=thin" does not work correctly for wasm-ld.
# we don't set onnxruntime_ENABLE_LTO because it appends flag "-flto=thin"
# instead, we manually set CMAKE_CXX_FLAGS "-flto"
string(APPEND CMAKE_CXX_FLAGS " -flto")
endif()

2
cmake/onnxruntime_webassembly.cmake
View file

@ -41,7 +41,7 @@ target_link_libraries(onnxruntime_webassembly PRIVATE
set(EXTRA_EXPORTED_RUNTIME_METHODS "['stackAlloc','stackRestore','stackSave','UTF8ToString','stringToUTF8','lengthBytesUTF8']")
set_target_properties(onnxruntime_webassembly PROPERTIES LINK_FLAGS " \
set_target_properties(onnxruntime_webassembly PROPERTIES LINK_FLAGS " \
-s \"EXTRA_EXPORTED_RUNTIME_METHODS=${EXTRA_EXPORTED_RUNTIME_METHODS}\" \
-s WASM=1 \
-s NO_EXIT_RUNTIME=0 \

8
js/.eslintrc.js
View file

@ -14,7 +14,9 @@ module.exports = {
], 2
],
'import/no-extraneous-dependencies': ['error', { 'devDependencies': false }],
'import/no-internal-modules': 'error',
'import/no-internal-modules': ['error', {
'allow': ['**/lib/**'],
}],
'import/no-unassigned-import': 'error',
'@typescript-eslint/array-type': ['error', { 'default': 'array-simple' }],
'@typescript-eslint/await-thenable': 'error',
@ -43,7 +45,7 @@ module.exports = {
'@typescript-eslint/no-inferrable-types': 'error',
'@typescript-eslint/no-misused-new': 'error',
'@typescript-eslint/no-namespace': ['error', { 'allowDeclarations': true }],
'@typescript-eslint/no-non-null-assertion': 'error',
'@typescript-eslint/no-non-null-assertion': 'off',
'@typescript-eslint/no-require-imports': 'error',
'@typescript-eslint/no-unnecessary-type-assertion': 'error',
'@typescript-eslint/no-unused-vars': ['error', { 'argsIgnorePattern': '^_' }],
@ -121,7 +123,6 @@ module.exports = {
'@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/no-require-imports': 'off',
'@typescript-eslint/no-var-requires': 'off',
'@typescript-eslint/no-non-null-assertion': 'off',
'@typescript-eslint/no-unnecessary-type-assertion': 'off',
'camelcase': 'off',
'prefer-arrow/prefer-arrow-functions': 'off',
@ -141,7 +142,6 @@ module.exports = {
// TODO: those rules are useful. should turn on them in future (webgl refactor)
'@typescript-eslint/no-empty-function': 'off',
'@typescript-eslint/explicit-module-boundary-types': 'off',
'@typescript-eslint/no-non-null-assertion': 'off',
'@typescript-eslint/no-use-before-define': 'off',
'@typescript-eslint/no-unnecessary-type-assertion': 'off',
'@typescript-eslint/restrict-plus-operands': 'off',

3
js/.vscode/settings.json vendored
View file

@ -36,5 +36,6 @@
"**/node_modules": true,
"./types": true
},
"typescript.tsdk": "node_modules/typescript/lib"
"typescript.tsdk": "node_modules/typescript/lib",
"git.detectSubmodules": false
}

2
js/common/lib/backend-impl.ts
View file

@ -80,5 +80,5 @@ export const resolveBackend = async(backendHints: readonly string[]): Promise<Ba
}
}
throw new Error(`no available backend found. ERR: ${errors.join(', ')}`);
throw new Error(`no available backend found. ERR: ${errors.map(e => `[${e.name}] ${e.err}`).join(', ')}`);
};

54
js/common/lib/env.ts
View file

@ -1,16 +1,66 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
interface Env {
export declare namespace Env {
export interface WebAssemblyFlags {
/**
* set or get number of thread(s). If omitted or set to 0, number of thread(s) will be determined by system. If set
* to 1, no worker thread will be spawned.
*
* This setting is available only when WebAssembly multithread feature is available in current context.
*/
numThreads?: number;
/**
* Set or get a number specifying the timeout for initialization of WebAssembly backend, in milliseconds. A zero
* value indicates no timeout is set. (default is 0)
*/
initTimeout?: number;
}
export interface WebGLFlags {
/**
* Set or get the WebGL Context ID (webgl or webgl2). Default value is webgl2.
*/
contextId?: 'webgl'|'webgl2';
/**
* Set or get the maximum batch size for matmul. 0 means to disable batching.
*/
matmulMaxBatchSize?: number;
/**
* Set or get the texture cache mode. Default value is full.
*/
textureCacheMode?: 'initializerOnly'|'full';
/**
* Set or get the packed texture mode
*/
pack?: boolean;
}
}
export interface Env {
/**
* Indicate whether run in debug mode.
*/
debug?: boolean;
/**
* Represent a set of flags for WebAssembly
*/
wasm: Env.WebAssemblyFlags;
/**
* Represent a set of flags for WebGL
*/
webgl: Env.WebGLFlags;
[name: string]: unknown;
}
/**
* Represent a set of flags as a global singleton.
*/
export const env: Env = {};
export const env: Env = {
wasm: {},
webgl: {}
};

4
js/common/lib/inference-session.ts
View file

@ -54,14 +54,14 @@ export declare namespace InferenceSession {
/**
* The intra OP threads number.
*
* This setting is available only in ONNXRuntime (Node.js binding and react-native) or WebAssembly backend
* This setting is available only in ONNXRuntime (Node.js binding and react-native).
*/
intraOpNumThreads?: number;
/**
* The inter OP threads number.
*
* This setting is available only in ONNXRuntime (Node.js binding and react-native) or WebAssembly backend
* This setting is available only in ONNXRuntime (Node.js binding and react-native).
*/
interOpNumThreads?: number;

3
js/web/karma.conf.js
View file

@ -3,6 +3,7 @@
const bundleMode = require('minimist')(process.argv)['bundle-mode'] || 'dev'; // 'dev'|'perf'|undefined;
const karmaPlugins = require('minimist')(process.argv)['karma-plugins'] || undefined;
const timeoutMocha = require('minimist')(process.argv)['timeout-mocha'] || 60000;
const commonFile = bundleMode === 'dev' ? '../common/dist/ort-common.js' : '../common/dist/ort-common.min.js'
const mainFile = bundleMode === 'dev' ? 'test/ort.dev.js' : 'test/ort.perf.js';
@ -59,7 +60,7 @@ module.exports = function (config) {
'/base/test/ort-wasm-threaded.worker.js': '/base/dist/ort-wasm-threaded.worker.js',
},
plugins: karmaPlugins,
client: { captureConsole: true, mocha: { expose: ['body'], timeout: 60000 } },
client: { captureConsole: true, mocha: { expose: ['body'], timeout: timeoutMocha } },
preprocessors: { mainFile: ['sourcemap'] },
reporters: ['mocha', 'BrowserStack'],
browsers: [],

26
js/web/lib/backend-onnxjs.ts
View file

@ -2,7 +2,7 @@
// Licensed under the MIT License.
/* eslint-disable import/no-internal-modules */
import {Backend, env, InferenceSession, SessionHandler} from 'onnxruntime-common';
import {Backend, InferenceSession, SessionHandler} from 'onnxruntime-common';
import {Session} from './onnxjs/session';
import {OnnxjsSessionHandler} from './onnxjs/session-handler';
@ -30,27 +30,3 @@ class OnnxjsBackend implements Backend {
}
export const onnxjsBackend = new OnnxjsBackend();
export interface WebGLFlags {
/**
* set or get the WebGL Context ID (webgl or webgl2)
*/
contextId?: 'webgl'|'webgl2';
/**
* set or get the maximum batch size for matmul. 0 means to disable batching.
*/
matmulMaxBatchSize?: number;
/**
* set or get the texture cache mode
*/
textureCacheMode?: 'initializerOnly'|'full';
/**
* set or get the packed texture mode
*/
pack?: boolean;
}
/**
* Represent a set of flags for ONNX.js backend.
*/
export const flags: WebGLFlags = env.webgl = env.webgl as WebGLFlags || {};

45
js/web/lib/backend-wasm.ts
View file

@ -3,11 +3,33 @@
import {Backend, env, InferenceSession, SessionHandler} from 'onnxruntime-common';
import {init, OnnxruntimeWebAssemblySessionHandler} from './wasm';
import {OnnxruntimeWebAssemblySessionHandler} from './wasm/session-handler';
import {initializeWebAssembly} from './wasm/wasm-factory';
/**
* This function initializes all flags for WebAssembly.
*
* Those flags are accessible from `ort.env.wasm`. Users are allow to set those flags before the first inference session
* being created, to override default value.
*/
export const initializeFlags = (): void => {
if (typeof env.wasm.initTimeout !== 'number' || env.wasm.initTimeout < 0) {
env.wasm.initTimeout = 0;
}
if (typeof env.wasm.numThreads !== 'number' || !Number.isInteger(env.wasm.numThreads) || env.wasm.numThreads < 0) {
env.wasm.numThreads = Math.ceil((navigator.hardwareConcurrency || 1) / 2);
}
env.wasm.numThreads = Math.min(4, env.wasm.numThreads);
};
class OnnxruntimeWebAssemblyBackend implements Backend {
async init(): Promise<void> {
await init();
// populate wasm flags
initializeFlags();
// init wasm
await initializeWebAssembly();
}
createSessionHandler(path: string, options?: InferenceSession.SessionOptions): Promise<SessionHandler>;
createSessionHandler(buffer: Uint8Array, options?: InferenceSession.SessionOptions): Promise<SessionHandler>;
@ -29,22 +51,3 @@ class OnnxruntimeWebAssemblyBackend implements Backend {
}
export const wasmBackend = new OnnxruntimeWebAssemblyBackend();
export interface WebAssemblyFlags {
/**
* set or get number of worker(s)
*
* This setting is available only when WebAssembly multithread feature is available in current context.
*/
worker?: number;
/**
* set or get a number specifying the timeout for initialization of WebAssembly backend, in milliseconds.
*/
initTimeout?: number;
}
/**
* Represent a set of flags for WebAssembly backend.
*/
export const flags: WebAssemblyFlags = env.wasm = env.wasm as WebAssemblyFlags || {};

18
js/web/lib/onnxjs/backends/backend-webgl.ts
View file

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {flags} from '../../backend-onnxjs';
import {env} from 'onnxruntime-common';
import {Backend, SessionHandler} from '../backend';
import {Logger} from '../instrument';
import {Session} from '../session';
@ -19,31 +19,31 @@ export class WebGLBackend implements Backend {
glContext: WebGLContext;
get contextId(): 'webgl'|'webgl2'|undefined {
return flags.contextId;
return env.webgl.contextId;
}
set contextId(value: 'webgl'|'webgl2'|undefined) {
flags.contextId = value;
env.webgl.contextId = value;
}
get matmulMaxBatchSize(): number|undefined {
return flags.matmulMaxBatchSize;
return env.webgl.matmulMaxBatchSize;
}
set matmulMaxBatchSize(value: number|undefined) {
flags.matmulMaxBatchSize = value;
env.webgl.matmulMaxBatchSize = value;
}
get textureCacheMode(): 'initializerOnly'|'full'|undefined {
return flags.textureCacheMode;
return env.webgl.textureCacheMode;
}
set textureCacheMode(value: 'initializerOnly'|'full'|undefined) {
flags.textureCacheMode = value;
env.webgl.textureCacheMode = value;
}
get pack(): boolean|undefined {
return flags.pack;
return env.webgl.pack;
}
set pack(value: boolean|undefined) {
flags.pack = value;
env.webgl.pack = value;
}
initialize(): boolean {

1
js/web/lib/onnxjs/instrument.ts
View file

@ -202,7 +202,6 @@ namespace log {
export function reset(config?: Logger.Config): void {
LOGGER_CONFIG_MAP = {};
// tslint:disable-next-line:no-backbone-get-set-outside-model
set('', config || {});
}
export function set(category: string, config: Logger.Config): void {

42
js/web/lib/wasm/binding/index.ts
View file

@ -1,42 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import wasmModuleFactory, {BackendWasmModule} from './ort-wasm';
// some global parameters to deal with wasm binding initialization
let wasm: BackendWasmModule;
let initialized = false;
let initializing = false;
/**
* initialize the WASM instance.
*
* this function should be called before any other calls to the WASM binding.
*/
export const init = async(): Promise<void> => {
if (initialized) {
return Promise.resolve();
}
if (initializing) {
throw new Error('multiple calls to \'init()\' detected.');
}
initializing = true;
return new Promise<void>((resolve, reject) => {
wasmModuleFactory().then(
initializedModule => {
// resolve init() promise
wasm = initializedModule;
initializing = false;
initialized = true;
resolve();
},
err => {
initializing = false;
reject(err);
});
});
};
export const getInstance = (): BackendWasmModule => wasm;

7
js/web/lib/wasm/binding/ort-wasm-threaded.d.ts vendored Normal file
View file

@ -0,0 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {OrtWasmModule} from './ort-wasm';
declare const moduleFactory: EmscriptenModuleFactory<OrtWasmModule>;
export default moduleFactory;

14
js/web/lib/wasm/binding/ort-wasm.d.ts vendored
View file

@ -1,7 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
export interface BackendWasmModule extends EmscriptenModule {
export interface OrtWasmModule extends EmscriptenModule {
//#region emscripten functions
stackSave(): number;
stackRestore(stack: number): void;
stackAlloc(size: number): number;
@ -9,8 +10,10 @@ export interface BackendWasmModule extends EmscriptenModule {
UTF8ToString(offset: number): string;
lengthBytesUTF8(str: string): number;
stringToUTF8(str: string, offset: number, maxBytes: number): void;
//#endregion
_OrtInit(): void;
//#region ORT APIs
_OrtInit(numThreads: number, loggingLevel: number): void;
_OrtCreateSession(dataOffset: number, dataLength: number): number;
_OrtReleaseSession(sessionHandle: number): void;
@ -29,7 +32,12 @@ export interface BackendWasmModule extends EmscriptenModule {
_OrtRun(
sessionHandle: number, inputNamesOffset: number, inputsOffset: number, inputCount: number,
outputNamesOffset: number, outputCount: number, outputsOffset: number): number;
//#endregion
//#region config
mainScriptUrlOrBlob?: string|Blob;
//#endregion
}
declare const moduleFactory: EmscriptenModuleFactory<BackendWasmModule>;
declare const moduleFactory: EmscriptenModuleFactory<OrtWasmModule>;
export default moduleFactory;

5
js/web/lib/wasm/index.ts
View file

@ -1,5 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
export {init} from './binding';
export {OnnxruntimeWebAssemblySessionHandler} from './session-handler';

6
js/web/lib/wasm/session-handler.ts
View file

@ -2,8 +2,8 @@
// Licensed under the MIT License.
import {onnx} from 'onnx-proto';
import {InferenceSession, SessionHandler, Tensor, TypedTensor} from 'onnxruntime-common';
import {getInstance} from './binding';
import {env, InferenceSession, SessionHandler, Tensor, TypedTensor} from 'onnxruntime-common';
import {getInstance} from './wasm-factory';
let ortInit: boolean;
@ -113,7 +113,7 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
loadModel(model: Uint8Array): void {
const wasm = getInstance();
if (!ortInit) {
wasm._OrtInit();
wasm._OrtInit(env.wasm.numThreads!, 2 /* LoggingLevel::Warning */);
ortInit = true;
}

102
js/web/lib/wasm/wasm-factory.ts Normal file
View file

@ -0,0 +1,102 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {env} from 'onnxruntime-common';
import {OrtWasmModule} from './binding/ort-wasm';
import ortWasmFactoryThreaded from './binding/ort-wasm-threaded.js';
import ortWasmFactory from './binding/ort-wasm.js';
let wasm: OrtWasmModule;
let initialized = false;
let initializing = false;
let aborted = false;
const isMultiThreadSupported = (): boolean => {
try {
// Test for transferability of SABs (needed for Firefox)
// https://groups.google.com/forum/#!msg/mozilla.dev.platform/IHkBZlHETpA/dwsMNchWEQAJ
new MessageChannel().port1.postMessage(new SharedArrayBuffer(1));
// This typed array is a WebAssembly program containing threaded
// instructions.
return WebAssembly.validate(new Uint8Array([
0, 97, 115, 109, 1, 0, 0, 0, 1, 4, 1, 96, 0, 0, 3, 2, 1, 0, 5,
4, 1, 3, 1, 1, 10, 11, 1, 9, 0, 65, 0, 254, 16, 2, 0, 26, 11
]));
} catch (e) {
return false;
}
};
export const initializeWebAssembly = async(): Promise<void> => {
if (initialized) {
return Promise.resolve();
}
if (initializing) {
throw new Error('multiple calls to \'initializeWebAssembly()\' detected.');
}
if (aborted) {
throw new Error('previous call to \'initializeWebAssembly()\' failed.');
}
initializing = true;
// wasm flags are already initialized
const timeout = env.wasm.initTimeout!;
const numThreads = env.wasm.numThreads!;
const useThreads = numThreads > 1 && isMultiThreadSupported();
let isTimeout = false;
const tasks: Array<Promise<void>> = [];
// promise for timeout
if (timeout > 0) {
tasks.push(new Promise((resolve) => {
setTimeout(() => {
isTimeout = true;
resolve();
}, timeout);
}));
}
// promise for module initialization
tasks.push(new Promise((resolve, reject) => {
const factory = useThreads ? ortWasmFactoryThreaded : ortWasmFactory;
const config: Partial<OrtWasmModule> = {};
if (useThreads) {
config.mainScriptUrlOrBlob = new Blob(
[`var ortWasmThreaded=(function(){var _scriptDir;return ${ortWasmFactoryThreaded.toString()}})();`],
{type: 'text/javascript'});
}
factory(config).then(
// wasm module initialized successfully
module => {
initializing = false;
initialized = true;
wasm = module;
resolve();
},
// wasm module failed to initialize
(what) => {
initializing = false;
aborted = true;
reject(what);
});
}));
await Promise.race(tasks);
if (isTimeout) {
throw new Error(`WebAssembly backend initializing failed due to timeout: ${timeout}ms`);
}
};
export const getInstance = (): OrtWasmModule => {
if (initialized) {
return wasm;
}
throw new Error('WebAssembly is not initialized yet.');
};

5
js/web/package.json
View file

@ -75,6 +75,9 @@
"browser": {
"fs": false,
"path": false,
"util": false
"util": false,
"worker_threads": false,
"perf_hooks": false,
"os": false
}
}

48
js/web/script/build.ts
View file

@ -9,29 +9,51 @@ import * as path from 'path';
// CMD args
const args = minimist(process.argv);
const MODE = args.config || 'prod'; // prod|dev|test
if (['prod', 'dev', 'test'].indexOf(MODE) === -1) {
// --bundle-mode=prod (default)
// --bundle-mode=dev
// --bundle-mode=perf
const MODE = args['bundle-mode'] || 'prod';
if (['prod', 'dev', 'perf'].indexOf(MODE) === -1) {
throw new Error(`unknown build mode: ${MODE}`);
}
// --wasm (default)
// --no-wasm
const WASM = typeof args.wasm === 'undefined' ? true : !!args.wasm;
// Path variables
const WASM_BINDING_FOLDER = path.join(__dirname, '..', 'lib', 'wasm', 'binding');
const WASM_JS_PATH = path.join(WASM_BINDING_FOLDER, 'ort-wasm.js');
const WASM_THREADED_JS_PATH = path.join(WASM_BINDING_FOLDER, 'ort-wasm-threaded.js');
const WASM_DIST_FOLDER = path.join(__dirname, '..', 'dist');
const WASM_DIST_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm.wasm');
const WASM_WASM_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm.wasm');
const WASM_THREADED_WASM_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm-threaded.wasm');
const WASM_THREADED_WORKER_JS_PATH = path.join(WASM_DIST_FOLDER, 'ort-wasm-threaded.worker.js');
try {
npmlog.info('Build', `Ensure file: ${WASM_JS_PATH}`);
if (!fs.pathExistsSync(WASM_JS_PATH)) {
throw new Error(`file does not exist: ${WASM_JS_PATH}`);
function validateFile(path: string): void {
npmlog.info('Build', `Ensure file: ${path}`);
if (!fs.pathExistsSync(path)) {
throw new Error(`file does not exist: ${path}`);
}
npmlog.info('Build', `Ensure file: ${WASM_DIST_PATH}`);
if (!fs.pathExistsSync(WASM_DIST_PATH)) {
throw new Error(`file does not exist: ${WASM_DIST_PATH}`);
if (fs.statSync(path).size === 0) {
throw new Error(`file is empty: ${path}`);
}
} catch (e) {
npmlog.error('Build', `WebAssembly files are not ready. build WASM first. ERR: ${e}`);
throw e;
}
if (WASM) {
npmlog.info('Build', 'Validating WebAssembly artifacts...');
try {
validateFile(WASM_JS_PATH);
validateFile(WASM_THREADED_JS_PATH);
validateFile(WASM_WASM_PATH);
validateFile(WASM_THREADED_WASM_PATH);
validateFile(WASM_THREADED_WORKER_JS_PATH);
} catch (e) {
npmlog.error('Build', `WebAssembly files are not ready. build WASM first. ERR: ${e}`);
throw e;
}
npmlog.info('Build', 'Validating WebAssembly artifacts... DONE');
}
npmlog.info('Build', 'Building bundle...');

26
js/web/script/test-runner-cli-args.ts
View file

@ -3,10 +3,8 @@
import minimist from 'minimist';
import npmlog from 'npmlog';
import {InferenceSession} from 'onnxruntime-common';
import {Env, InferenceSession} from 'onnxruntime-common';
import {WebGLFlags} from '../lib/backend-onnxjs';
import {WebAssemblyFlags} from '../lib/backend-wasm';
import {Logger} from '../lib/onnxjs/instrument';
import {Test} from '../test/test-types';
@ -14,7 +12,7 @@ import {Test} from '../test/test-types';
const HELP_MESSAGE = `
test-runner-cli
Run ONNX.js tests, models, benchmarks in different environments.
Run ONNX Runtime Web tests, models, benchmarks in different environments.
Usage:
test-runner-cli <mode> ... [options]
@ -31,7 +29,7 @@ Options:
-h, --help Print this message.
-d, --debug Specify to run test runner in debug mode.
Debug mode outputs verbose log for test runner, sets up ONNX.js environment debug flag, and keeps karma not to exit after tests completed.
Debug mode outputs verbose log for test runner, sets up environment debug flag, and keeps karma not to exit after tests completed.
-b=<...>, --backend=<...> Specify one or more backend(s) to run the test upon.
Backends can be one or more of the following, splitted by comma:
webgl
@ -60,7 +58,7 @@ Options:
*** Backend Options ***
--wasm-worker Set the WebAssembly worker number
--wasm-number-threads Set the WebAssembly number of threads
--wasm-init-timeout Set the timeout for WebAssembly backend initialization, in milliseconds
--webgl-context-id Set the WebGL context ID (webgl/webgl2)
--webgl-matmul-max-batch-size Set the WebGL matmulMaxBatchSize
@ -151,9 +149,9 @@ export interface TestRunnerCliArgs {
cudaOptions?: InferenceSession.CudaExecutionProviderOption;
cudaFlags?: Record<string, unknown>;
wasmOptions?: InferenceSession.WebAssemblyExecutionProviderOption;
wasmFlags?: WebAssemblyFlags;
wasmFlags?: Env.WebAssemblyFlags;
webglOptions?: InferenceSession.WebGLExecutionProviderOption;
webglFlags?: WebGLFlags;
webglFlags?: Env.WebGLFlags;
noSandbox?: boolean;
}
@ -246,23 +244,23 @@ function parseWasmOptions(_args: minimist.ParsedArgs): InferenceSession.WebAssem
return {name: 'wasm'};
}
function parseWasmFlags(args: minimist.ParsedArgs): WebAssemblyFlags {
const worker = args['wasm-worker'];
if (typeof worker !== 'undefined' && typeof worker !== 'number') {
throw new Error('Flag "wasm-worker" must be a number value');
function parseWasmFlags(args: minimist.ParsedArgs): Env.WebAssemblyFlags {
const numThreads = args['wasm-number-threads'];
if (typeof numThreads !== 'undefined' && typeof numThreads !== 'number') {
throw new Error('Flag "wasm-number-threads" must be a number value');
}
const initTimeout = args['wasm-init-timeout'];
if (typeof initTimeout !== 'undefined' && typeof initTimeout !== 'number') {
throw new Error('Flag "wasm-init-timeout" must be a number value');
}
return {worker, initTimeout};
return {numThreads, initTimeout};
}
function parseWebglOptions(_args: minimist.ParsedArgs): InferenceSession.WebGLExecutionProviderOption {
return {name: 'webgl'};
}
function parseWebglFlags(args: minimist.ParsedArgs): WebGLFlags {
function parseWebglFlags(args: minimist.ParsedArgs): Env.WebGLFlags {
const contextId = args['webgl-context-id'];
if (contextId !== undefined && contextId !== 'webgl' && contextId !== 'webgl2') {
throw new Error('Flag "webgl-context-id" is invalid');

36
js/web/script/test-runner-cli.ts
View file

@ -410,7 +410,7 @@ function run(config: Test.Config) {
npmlog.info('TestRunnerCli.Run', `(3/5) Retrieving npm bin folder... DONE, folder: ${npmBin}`);
if (args.env === 'node') {
// STEP 4. use tsc to build ONNX.js
// STEP 4. use tsc to build ONNX Runtime Web
npmlog.info('TestRunnerCli.Run', '(4/5) Running tsc...');
const tscCommand = path.join(npmBin, 'tsc');
const tsc = spawnSync(tscCommand, {shell: true, stdio: 'inherit'});
@ -433,17 +433,20 @@ function run(config: Test.Config) {
npmlog.info('TestRunnerCli.Run', '(5/5) Running mocha... DONE');
} else {
// STEP 4. use webpack to generate ONNX.js
npmlog.info('TestRunnerCli.Run', '(4/5) Running webpack to generate ONNX.js...');
const webpackCommand = path.join(npmBin, 'webpack');
const webpackArgs = ['--env', `--bundle-mode=${args.bundleMode}`];
npmlog.info('TestRunnerCli.Run', `CMD: ${webpackCommand} ${webpackArgs.join(' ')}`);
const webpack = spawnSync(webpackCommand, webpackArgs, {shell: true, stdio: 'inherit'});
if (webpack.status !== 0) {
console.error(webpack.error);
process.exit(webpack.status === null ? undefined : webpack.status);
// STEP 4. generate bundle
npmlog.info('TestRunnerCli.Run', '(4/5) Running build to generate bundle...');
const buildCommand = `node ${path.join(__dirname, 'build')}`;
const buildArgs = [`--bundle-mode=${args.bundleMode}`];
if (args.backends.indexOf('wasm') === -1) {
buildArgs.push('--no-wasm');
}
npmlog.info('TestRunnerCli.Run', '(4/5) Running webpack to generate ONNX.js... DONE');
npmlog.info('TestRunnerCli.Run', `CMD: ${buildCommand} ${buildArgs.join(' ')}`);
const build = spawnSync(buildCommand, buildArgs, {shell: true, stdio: 'inherit'});
if (build.status !== 0) {
console.error(build.error);
process.exit(build.status === null ? undefined : build.status);
}
npmlog.info('TestRunnerCli.Run', '(4/5) Running build to generate bundle... DONE');
// STEP 5. use Karma to run test
npmlog.info('TestRunnerCli.Run', '(5/5) Running karma to start test runner...');
@ -451,7 +454,7 @@ function run(config: Test.Config) {
const browser = getBrowserNameFromEnv(args.env, args.debug);
const karmaArgs = ['start', `--browsers ${browser}`];
if (args.debug) {
karmaArgs.push('--log-level info');
karmaArgs.push('--log-level info --timeout-mocha 9999999');
} else {
karmaArgs.push('--single-run');
}
@ -475,7 +478,6 @@ function run(config: Test.Config) {
// check if we have the latest Edge installed:
if (os.platform() === 'darwin' ||
(os.platform() === 'win32' &&
// tslint:disable-next-line:no-require-imports no-submodule-imports
require('@chiragrupani/karma-chromium-edge-launcher/dist/Utilities').default.GetEdgeExe('Edge') !== '')) {
// use "@chiragrupani/karma-chromium-edge-launcher"
karmaArgs.push(
@ -563,15 +565,15 @@ function saveConfig(config: Test.Config) {
if (config.options.webglFlags && config.options.webglFlags.pack !== undefined) {
setOptions += `ort.env.webgl.pack = ${JSON.stringify(config.options.webglFlags.pack)};`;
}
if (config.options.wasmFlags && config.options.wasmFlags.worker !== undefined) {
setOptions += `ort.env.wasm.worker = ${JSON.stringify(config.options.wasmFlags.worker)};`;
if (config.options.wasmFlags && config.options.wasmFlags.numThreads !== undefined) {
setOptions += `ort.env.wasm.numThreads = ${JSON.stringify(config.options.wasmFlags.numThreads)};`;
}
if (config.options.wasmFlags && config.options.wasmFlags.initTimeout !== undefined) {
setOptions += `ort.env.wasm.initTimeout = ${JSON.stringify(config.options.wasmFlags.initTimeout)};`;
}
// TODO: support onnxruntime nodejs binding
// if (config.model.some(testGroup => testGroup.tests.some(test => test.backend === 'onnxruntime'))) {
// setOptions += 'require(\'onnxjs-node\');';
// if (config.model.some(testGroup => testGroup.tests.some(test => test.backend === 'cpu'))) {
// setOptions += 'require(\'onnxruntime-node\');';
// }
fs.writeFileSync(path.join(TEST_ROOT, './testdata-config.js'), `${setOptions}

21
js/web/test/test-main.ts
View file

@ -10,10 +10,10 @@ import {Logger} from '../lib/onnxjs/instrument';
import {Test} from './test-types';
const ONNX_JS_TEST_CONFIG = (ort.env as any).ORT_WEB_TEST_DATA as Test.Config;
const ORT_WEB_TEST_CONFIG = (ort.env as any).ORT_WEB_TEST_DATA as Test.Config;
// Set logging configuration
for (const logConfig of ONNX_JS_TEST_CONFIG.log) {
for (const logConfig of ORT_WEB_TEST_CONFIG.log) {
Logger.set(logConfig.category, logConfig.config);
}
@ -21,15 +21,14 @@ import {ModelTestContext, OpTestContext, runModelTestSet, runOpTest} from './tes
import {readJsonFile} from './test-shared';
// Unit test
if (ONNX_JS_TEST_CONFIG.unittest) {
// tslint:disable-next-line:no-require-imports
if (ORT_WEB_TEST_CONFIG.unittest) {
require('./unittests');
}
// Set file cache
if (ONNX_JS_TEST_CONFIG.fileCacheUrls) {
if (ORT_WEB_TEST_CONFIG.fileCacheUrls) {
before('prepare file cache', async () => {
const allJsonCache = await Promise.all(ONNX_JS_TEST_CONFIG.fileCacheUrls!.map(readJsonFile)) as Test.FileCache[];
const allJsonCache = await Promise.all(ORT_WEB_TEST_CONFIG.fileCacheUrls!.map(readJsonFile)) as Test.FileCache[];
for (const cache of allJsonCache) {
ModelTestContext.setCache(cache);
}
@ -52,7 +51,7 @@ function shouldSkipTest(test: Test.ModelTest|Test.OperatorTest) {
}
// ModelTests
for (const group of ONNX_JS_TEST_CONFIG.model) {
for (const group of ORT_WEB_TEST_CONFIG.model) {
describe(`#ModelTest# - ${group.name}`, () => {
for (const test of group.tests) {
const describeTest = shouldSkipTest(test) ? describe.skip : describe;
@ -60,7 +59,7 @@ for (const group of ONNX_JS_TEST_CONFIG.model) {
let context: ModelTestContext;
before('prepare session', async () => {
context = await ModelTestContext.create(test, ONNX_JS_TEST_CONFIG.profile);
context = await ModelTestContext.create(test, ORT_WEB_TEST_CONFIG.profile);
});
after('release session', () => {
@ -80,7 +79,7 @@ for (const group of ONNX_JS_TEST_CONFIG.model) {
}
// OpTests
for (const group of ONNX_JS_TEST_CONFIG.op) {
for (const group of ORT_WEB_TEST_CONFIG.op) {
describe(`#OpTest# - ${group.name}`, () => {
for (const test of group.tests) {
const describeTest = shouldSkipTest(test) ? describe.skip : describe;
@ -90,13 +89,13 @@ for (const group of ONNX_JS_TEST_CONFIG.op) {
before('Initialize Context', async () => {
context = new OpTestContext(test);
await context.init();
if (ONNX_JS_TEST_CONFIG.profile) {
if (ORT_WEB_TEST_CONFIG.profile) {
OpTestContext.profiler.start();
}
});
after('Dispose Context', () => {
if (ONNX_JS_TEST_CONFIG.profile) {
if (ORT_WEB_TEST_CONFIG.profile) {
OpTestContext.profiler.stop();
}
context.dispose();

19
js/web/test/test-runner.ts
View file

@ -7,7 +7,6 @@ import {onnx as onnxProto} from 'onnx-proto';
import * as ort from 'onnxruntime-common';
import {extname} from 'path';
import {inspect, promisify} from 'util';
import {flags as webglFlags} from '../lib/backend-onnxjs';
import {Attribute} from '../lib/onnxjs/attribute';
import {InferenceHandler, resolveBackend, SessionHandler} from '../lib/onnxjs/backend';
@ -69,7 +68,9 @@ async function loadMlProto(_uriOrData: string|Uint8Array): Promise<Test.NamedTen
return Promise.reject('not supported');
}
async function loadTensors(testCase: Test.ModelTestCase, fileCache?: FileCacheBuffer) {
async function loadTensors(
modelMetaData: {inputNames: readonly string[]; outputNames: readonly string[]}, testCase: Test.ModelTestCase,
fileCache?: FileCacheBuffer) {
const inputs: Test.NamedTensor[] = [];
const outputs: Test.NamedTensor[] = [];
let dataFileType: 'none'|'pb'|'npy' = 'none';
@ -100,6 +101,14 @@ async function loadTensors(testCase: Test.ModelTestCase, fileCache?: FileCacheBu
}
}
// if model has single input/output, and tensor name is empty, we assign model's input/output names to it.
if (modelMetaData.inputNames.length === 1 && inputs.length === 1 && !inputs[0].name) {
inputs[0].name = modelMetaData.inputNames[0];
}
if (modelMetaData.outputNames.length === 1 && outputs.length === 1 && !outputs[0].name) {
outputs[0].name = modelMetaData.outputNames[0];
}
testCase.inputs = inputs;
testCase.outputs = outputs;
}
@ -199,7 +208,7 @@ export class ModelTestContext {
const initEnd = now();
for (const testCase of modelTest.cases) {
await loadTensors(testCase, this.cache);
await loadTensors(session, testCase, this.cache);
}
return new ModelTestContext(
@ -250,7 +259,7 @@ export class TensorResultValidator {
this.relativeThreshold = CPU_THRESHOLD_RELATIVE_ERROR;
} else if (backend === 'webgl') {
if (TensorResultValidator.isHalfFloat === undefined) {
TensorResultValidator.isHalfFloat = !createWebGLContext(webglFlags.contextId).isRenderFloat32Supported;
TensorResultValidator.isHalfFloat = !createWebGLContext(ort.env.webgl.contextId).isRenderFloat32Supported;
}
if (TensorResultValidator.isHalfFloat) {
this.maxFloatValue = 65504;
@ -457,7 +466,7 @@ export async function runModelTestSet(
try {
if (context.backend === 'webgl') {
// TODO skipping incompatible tests for now
if (createWebGLContext(webglFlags.contextId).version === 1 &&
if (createWebGLContext(ort.env.webgl.contextId).version === 1 &&
UNSUPPORTED_WEBGL_1_TESTS.indexOf(testName) !== -1) {
Logger.info('TestRunner', `Found incompatible test on webgl 1: ${testName} - ${testCase.name}. Skipping.`);
return;

10
js/web/test/test-types.ts
View file

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {InferenceSession, Tensor} from 'onnxruntime-common';
import {Env, InferenceSession, Tensor} from 'onnxruntime-common';
import {WebGLFlags} from '../lib/backend-onnxjs';
import {WebAssemblyFlags} from '../lib/backend-wasm';
import {Attribute} from '../lib/onnxjs/attribute';
import {Logger} from '../lib/onnxjs/instrument';
@ -102,7 +100,7 @@ export declare namespace Test {
}
/**
* Represent ONNX.js global options
* Represent ONNX Runtime Web global options
*/
export interface Options {
debug?: boolean;
@ -111,9 +109,9 @@ export declare namespace Test {
cudaOptions?: InferenceSession.CudaExecutionProviderOption;
cudaFlags?: Record<string, unknown>;
wasmOptions?: InferenceSession.WebAssemblyExecutionProviderOption;
wasmFlags?: WebAssemblyFlags;
wasmFlags?: Env.WebAssemblyFlags;
webglOptions?: InferenceSession.WebGLExecutionProviderOption;
webglFlags?: WebGLFlags;
webglFlags?: Env.WebGLFlags;
}
/**

1
js/web/test/unittests/opset.ts
View file

@ -14,7 +14,6 @@ function createTestGraphNode(name: string, opType: string): Graph.Node {
}
function dummyOpConstructor(): Operator {
// tslint:disable-next-line:no-any
return {} as any as Operator;
}

66
js/web/webpack.config.js
View file

@ -102,8 +102,14 @@ function buildTestRunnerConfig({
externals: {
'onnxruntime-common': 'ort',
'fs': 'fs',
'perf_hooks': 'perf_hooks',
'worker_threads': 'worker_threads',
},
resolve: {
extensions: ['.ts', '.js'],
aliasFields: [],
fallback: { './binding/ort-wasm-threaded.js': false, './binding/ort-wasm.js': false }
},
resolve: { extensions: ['.ts', '.js'], aliasFields: [] },
plugins: [
new webpack.WatchIgnorePlugin({ paths: [/\.js$/, /\.d\.ts$/] }),
new NodePolyfillPlugin()
@ -132,35 +138,39 @@ module.exports = () => {
const bundleMode = args['bundle-mode'] || 'prod'; // 'prod'|'dev'|'perf'|undefined;
const builds = [];
if (bundleMode === 'prod') {
builds.push(
// ort.min.js
buildOrtConfig({ suffix: '.min' }),
// ort.js
buildOrtConfig({ mode: 'development', devtool: 'inline-source-map' }),
// ort.es6.min.js
buildOrtConfig({ suffix: '.es6.min', target: 'es6' }),
// ort.es6.js
buildOrtConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }),
switch (bundleMode) {
case 'prod':
builds.push(
// ort.min.js
buildOrtConfig({ suffix: '.min' }),
// ort.js
buildOrtConfig({ mode: 'development', devtool: 'inline-source-map' }),
// ort.es6.min.js
buildOrtConfig({ suffix: '.es6.min', target: 'es6' }),
// ort.es6.js
buildOrtConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }),
// ort-web.min.js
buildOrtWebConfig({ suffix: '.min' }),
// ort-web.js
buildOrtWebConfig({ mode: 'development', devtool: 'inline-source-map' }),
// ort-web.es6.min.js
buildOrtWebConfig({ suffix: '.es6.min', target: 'es6' }),
// ort-web.es6.js
buildOrtWebConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }),
// ort-web.min.js
buildOrtWebConfig({ suffix: '.min' }),
// ort-web.js
buildOrtWebConfig({ mode: 'development', devtool: 'inline-source-map' }),
// ort-web.es6.min.js
buildOrtWebConfig({ suffix: '.es6.min', target: 'es6' }),
// ort-web.es6.js
buildOrtWebConfig({ suffix: '.es6', mode: 'development', devtool: 'inline-source-map', target: 'es6' }),
// ort-web.node.js
buildOrtWebConfig({ suffix: '.node', format: 'commonjs' }),
);
}
if (bundleMode === 'dev') {
builds.push(buildTestRunnerConfig({ suffix: '.dev', mode: 'development', devtool: 'inline-source-map' }));
} else if (bundleMode === 'perf') {
builds.push(buildTestRunnerConfig({ suffix: '.perf', devtool: undefined }));
// ort-web.node.js
buildOrtWebConfig({ suffix: '.node', format: 'commonjs' }),
);
break;
case 'dev':
builds.push(buildTestRunnerConfig({ suffix: '.dev', mode: 'development', devtool: 'inline-source-map' }));
break;
case 'perf':
builds.push(buildTestRunnerConfig({ suffix: '.perf' })); // TODO: .js.LICENSE.txt
break;
default:
throw new Error(`unsupported bundle mode: ${bundleMode}`);
}
return builds;

18
onnxruntime/wasm/api.cc
View file

@ -12,17 +12,25 @@ namespace {
Ort::Env* g_env;
} // namespace
void OrtInit() {
// TODO: allow user to specify logging
OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING;
g_env = new Ort::Env{logging_level, "Default"};
void OrtInit(int numThreads, int logging_level) {
#if defined(__EMSCRIPTEN_PTHREADS__)
OrtThreadingOptions* tp_options;
Ort::ThrowOnError(Ort::GetApi().CreateThreadingOptions(&tp_options));
Ort::ThrowOnError(Ort::GetApi().SetGlobalIntraOpNumThreads(tp_options, numThreads));
Ort::ThrowOnError(Ort::GetApi().SetGlobalInterOpNumThreads(tp_options, 1));
g_env = new Ort::Env{tp_options, static_cast<OrtLoggingLevel>(logging_level), "Default"};
#endif
g_env = new Ort::Env{static_cast<OrtLoggingLevel>(logging_level), "Default"};
}
Ort::Session* OrtCreateSession(void* data, size_t data_length) {
Ort::SessionOptions session_options;
session_options.SetLogId("onnxruntime");
#if !defined(__EMSCRIPTEN_PTHREADS__)
#if defined(__EMSCRIPTEN_PTHREADS__)
session_options.DisablePerSessionThreads();
#else
// must disable thread pool when WebAssembly multi-threads support is disabled.
session_options.SetIntraOpNumThreads(1);
#endif

4
onnxruntime/wasm/api.h
View file

@ -24,8 +24,10 @@ extern "C" {
/**
* perform global initialization. should be called only once.
* @param numThreads number of total threads to use.
* @param logging_level default logging level.
*/
void EMSCRIPTEN_KEEPALIVE OrtInit();
void EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level);
/**
* create an instance of ORT session.