From 1ab8a95eb6675afb6d0ad9d93600ef0022e2ddb5 Mon Sep 17 00:00:00 2001 From: Sunghoon <35605090+hanbitmyths@users.noreply.github.com> Date: Thu, 13 May 2021 10:50:04 -0700 Subject: [PATCH] Bind existing SessionOptions and RunOptions in Javascript API with WebAssembly (#7621) * support session options and run options. use onnxruntime c api. * fix lint errors * add an error code on throwing an exception * resolve comments. change remaining C++ APIs to C API --- js/common/lib/env.ts | 5 + js/web/lib/backend-wasm.ts | 11 +- js/web/lib/wasm/binding/ort-wasm.d.ts | 24 +- js/web/lib/wasm/session-handler.ts | 373 +++++++++++++++++++------- js/web/script/test-runner-cli.ts | 3 + onnxruntime/wasm/api.cc | 260 +++++++++++++----- onnxruntime/wasm/api.h | 95 ++++++- 7 files changed, 597 insertions(+), 174 deletions(-) diff --git a/js/common/lib/env.ts b/js/common/lib/env.ts index 8b09ed20ed..74d6e3ff2c 100644 --- a/js/common/lib/env.ts +++ b/js/common/lib/env.ts @@ -11,6 +11,11 @@ export declare namespace Env { */ numThreads?: number; + /** + * set a logging level. If omitted, default is 'warning' + */ + loggingLevel?: 'verbose'|'info'|'warning'|'error'|'fatal'; + /** * Set or get a number specifying the timeout for initialization of WebAssembly backend, in milliseconds. A zero * value indicates no timeout is set. (default is 0) diff --git a/js/web/lib/backend-wasm.ts b/js/web/lib/backend-wasm.ts index bebaffb19b..9192a2ebd6 100644 --- a/js/web/lib/backend-wasm.ts +++ b/js/web/lib/backend-wasm.ts @@ -21,6 +21,11 @@ export const initializeFlags = (): void => { env.wasm.numThreads = Math.ceil((navigator.hardwareConcurrency || 1) / 2); } env.wasm.numThreads = Math.min(4, env.wasm.numThreads); + + if (typeof env.wasm.loggingLevel !== 'string' || + ['verbose', 'info', 'warning', 'error', 'fatal'].indexOf(env.wasm.loggingLevel) === -1) { + env.wasm.loggingLevel = 'warning'; + } }; class OnnxruntimeWebAssemblyBackend implements Backend { @@ -33,7 +38,7 @@ class OnnxruntimeWebAssemblyBackend implements Backend { } createSessionHandler(path: string, options?: InferenceSession.SessionOptions): Promise; createSessionHandler(buffer: Uint8Array, options?: InferenceSession.SessionOptions): Promise; - async createSessionHandler(pathOrBuffer: string|Uint8Array, _options?: InferenceSession.SessionOptions): + async createSessionHandler(pathOrBuffer: string|Uint8Array, options?: InferenceSession.SessionOptions): Promise { let buffer: Uint8Array; if (typeof pathOrBuffer === 'string') { @@ -43,9 +48,9 @@ class OnnxruntimeWebAssemblyBackend implements Backend { } else { buffer = pathOrBuffer; } + const handler = new OnnxruntimeWebAssemblySessionHandler(); - // TODO: support SessionOptions - handler.loadModel(buffer); + handler.loadModel(buffer, options); return Promise.resolve(handler); } } diff --git a/js/web/lib/wasm/binding/ort-wasm.d.ts b/js/web/lib/wasm/binding/ort-wasm.d.ts index 2c12322c84..c8173b38e8 100644 --- a/js/web/lib/wasm/binding/ort-wasm.d.ts +++ b/js/web/lib/wasm/binding/ort-wasm.d.ts @@ -13,9 +13,9 @@ export interface OrtWasmModule extends EmscriptenModule { //#endregion //#region ORT APIs - _OrtInit(numThreads: number, loggingLevel: number): void; + _OrtInit(numThreads: number, loggingLevel: number): number; - _OrtCreateSession(dataOffset: number, dataLength: number): number; + _OrtCreateSession(dataOffset: number, dataLength: number, sessionOptionsHandle: number): number; _OrtReleaseSession(sessionHandle: number): void; _OrtGetInputCount(sessionHandle: number): number; _OrtGetOutputCount(sessionHandle: number): number; @@ -27,11 +27,27 @@ export interface OrtWasmModule extends EmscriptenModule { _OrtCreateTensor(dataType: number, dataOffset: number, dataLength: number, dimsOffset: number, dimsLength: number): number; _OrtGetTensorData(tensorHandle: number, dataType: number, dataOffset: number, dimsOffset: number, dimsLength: number): - void; + number; _OrtReleaseTensor(tensorHandle: number): void; _OrtRun( sessionHandle: number, inputNamesOffset: number, inputsOffset: number, inputCount: number, - outputNamesOffset: number, outputCount: number, outputsOffset: number): number; + outputNamesOffset: number, outputCount: number, outputsOffset: number, runOptionsHandle: number): number; + + _OrtCreateSessionOptions(): number; + _OrtReleaseSessionOptions(sessionOptionsHandle: number): void; + _OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle: number, level: number): number; + _OrtEnableCpuMemArena(sessionOptionsHandle: number): number; + _OrtDisableCpuMemArena(sessionOptionsHandle: number): number; + _OrtEnableMemPattern(sessionOptionsHandle: number): number; + _OrtDisableMemPattern(sessionOptionsHandle: number): number; + _OrtSetSessionExecutionMode(sessionOptionsHandle: number, mode: number): number; + _OrtSetSessionLogId(sessionOptionsHandle: number, logid: number): number; + _OrtSetSessionLogSeverityLevel(sessionOptionsHandle: number, level: number): number; + + _OrtCreateRunOptions(): number; + _OrtReleaseRunOptions(runOptionsHandle: number): void; + _OrtRunOptionsSetRunLogSeverityLevel(runOptionsHandle: number, level: number): number; + _OrtRunOptionsSetRunTag(runOptionsHandle: number, tag: number): number; //#endregion //#region config diff --git a/js/web/lib/wasm/session-handler.ts b/js/web/lib/wasm/session-handler.ts index d0edc73f2d..5dc562368a 100644 --- a/js/web/lib/wasm/session-handler.ts +++ b/js/web/lib/wasm/session-handler.ts @@ -102,6 +102,150 @@ const numericTensorTypeToTypedArray = (type: Tensor.Type): Float32ArrayConstruct } }; +const getLoggingLevel = (loggingLevel: 'verbose'|'info'|'warning'|'error'|'fatal'): number => { + switch (loggingLevel) { + case 'verbose': + return 0; + case 'info': + return 1; + case 'warning': + return 2; + case 'error': + return 3; + case 'fatal': + return 4; + default: + throw new Error(`unsupported logging level: ${loggingLevel}`); + } +}; + +const setSessionOptions = (options?: InferenceSession.SessionOptions): [number, number[]] => { + const wasm = getInstance(); + const sessionOptionsHandle = wasm._OrtCreateSessionOptions(); + const allocs: number[] = []; + + if (sessionOptionsHandle === 0) { + throw new Error('Can\'t create session options'); + } + + if (options === undefined) { + return [sessionOptionsHandle, allocs]; + } + + let errorCode = 0; + + if (options.graphOptimizationLevel !== undefined) { + switch (options.graphOptimizationLevel) { + case 'disabled': + errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 0); + break; + case 'basic': + errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 1); + break; + case 'extended': + errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 2); + break; + case 'all': + errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 99); + break; + default: + throw new Error(`unsupported graph optimization level: ${options.graphOptimizationLevel}`); + } + if (errorCode !== 0) { + throw new Error(`Can't set a graph optimization level as a session option. error code = ${errorCode}`); + } + } + + if (options.enableCpuMemArena !== undefined) { + if (options.enableCpuMemArena) { + errorCode = wasm._OrtEnableCpuMemArena(sessionOptionsHandle); + } else { + errorCode = wasm._OrtDisableCpuMemArena(sessionOptionsHandle); + } + if (errorCode !== 0) { + throw new Error(`Can't set a CPU memory arena as a session option. error code = ${errorCode}`); + } + } + + if (options.enableMemPattern !== undefined) { + if (options.enableMemPattern) { + errorCode = wasm._OrtEnableMemPattern(sessionOptionsHandle); + } else { + errorCode = wasm._OrtDisableMemPattern(sessionOptionsHandle); + } + if (errorCode !== 0) { + throw new Error(`Can't set a memory pattern as a session option. error code = ${errorCode}`); + } + } + + if (options.executionMode !== undefined) { + switch (options.executionMode) { + case 'sequential': + errorCode = wasm._OrtSetSessionExecutionMode(sessionOptionsHandle, 0); + break; + case 'parallel': + errorCode = wasm._OrtSetSessionExecutionMode(sessionOptionsHandle, 1); + break; + default: + throw new Error(`unsupported execution mode: ${options.executionMode}`); + } + if (errorCode !== 0) { + throw new Error(`Can't set an execution mode as a session option. error code = ${errorCode}`); + } + } + + if (options.logId !== undefined) { + const logIdDataLength = wasm.lengthBytesUTF8(options.logId) + 1; + const logIdDataOffset = wasm._malloc(logIdDataLength); + wasm.stringToUTF8(options.logId, logIdDataOffset, logIdDataLength); + errorCode = wasm._OrtSetSessionLogId(sessionOptionsHandle, logIdDataOffset); + allocs.push(logIdDataOffset); + if (errorCode !== 0) { + throw new Error(`Can't set a log id as a session option. error code = ${errorCode}`); + } + } + + if (options.logSeverityLevel !== undefined) { + errorCode = wasm._OrtSetSessionLogSeverityLevel(sessionOptionsHandle, options.logSeverityLevel); + if (errorCode !== 0) { + throw new Error(`Can't set a log severity level as a session option. error code = ${errorCode}`); + } + } + + return [sessionOptionsHandle, allocs]; +}; + +const setRunOptions = (options: InferenceSession.RunOptions): [number, number[]] => { + const wasm = getInstance(); + const runOptionsHandle = wasm._OrtCreateRunOptions(); + if (runOptionsHandle === 0) { + throw new Error('Can\'t create run options'); + } + + const allocs: number[] = []; + let errorCode = 0; + + if (options.logSeverityLevel !== undefined) { + errorCode = wasm._OrtRunOptionsSetRunLogSeverityLevel(runOptionsHandle, options.logSeverityLevel); + if (errorCode !== 0) { + throw new Error(`Can't set a log severity level as a run option. error code = ${errorCode}`); + } + } + + if (options.tag !== undefined) { + const tagDataLength = wasm.lengthBytesUTF8(options.tag) + 1; + const tagDataOffset = wasm._malloc(tagDataLength); + wasm.stringToUTF8(options.tag, tagDataOffset, tagDataLength); + errorCode = wasm._OrtRunOptionsSetRunTag(runOptionsHandle, tagDataOffset); + allocs.push(tagDataOffset); + if (errorCode !== 0) { + throw new Error(`Can't set a tag as a run option. error code = ${errorCode}`); + } + } + + return [runOptionsHandle, allocs]; +}; + export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { private sessionHandle: number; @@ -110,19 +254,32 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { outputNames: string[]; private outputNamesUTF8Encoded: number[]; - loadModel(model: Uint8Array): void { + loadModel(model: Uint8Array, options?: InferenceSession.SessionOptions): void { const wasm = getInstance(); if (!ortInit) { - wasm._OrtInit(env.wasm.numThreads!, 2 /* LoggingLevel::Warning */); + const errorCode = wasm._OrtInit(env.wasm.numThreads!, getLoggingLevel(env.wasm.loggingLevel!)); + if (errorCode !== 0) { + throw new Error(`Can't initialize onnxruntime. error code = ${errorCode}`); + } ortInit = true; } const modelDataOffset = wasm._malloc(model.byteLength); + let sessionOptionsHandle = 0; + let allocs: number[] = []; + try { + [sessionOptionsHandle, allocs] = setSessionOptions(options); + wasm.HEAPU8.set(model, modelDataOffset); - this.sessionHandle = wasm._OrtCreateSession(modelDataOffset, model.byteLength); + this.sessionHandle = wasm._OrtCreateSession(modelDataOffset, model.byteLength, sessionOptionsHandle); + if (this.sessionHandle === 0) { + throw new Error('Can\'t create a session'); + } } finally { wasm._free(modelDataOffset); + wasm._OrtReleaseSessionOptions(sessionOptionsHandle); + allocs.forEach(wasm._free); } const inputCount = wasm._OrtGetInputCount(this.sessionHandle); @@ -134,11 +291,17 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { this.outputNamesUTF8Encoded = []; for (let i = 0; i < inputCount; i++) { const name = wasm._OrtGetInputName(this.sessionHandle, i); + if (name === 0) { + throw new Error('Can\'t get an input name'); + } this.inputNamesUTF8Encoded.push(name); this.inputNames.push(wasm.UTF8ToString(name)); } for (let i = 0; i < outputCount; i++) { const name = wasm._OrtGetOutputName(this.sessionHandle, i); + if (name === 0) { + throw new Error('Can\'t get an output name'); + } this.outputNamesUTF8Encoded.push(name); this.outputNames.push(wasm.UTF8ToString(name)); } @@ -147,11 +310,11 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { async dispose(): Promise { const wasm = getInstance(); if (this.inputNamesUTF8Encoded) { - this.inputNamesUTF8Encoded.forEach(str => wasm._OrtFree(str)); + this.inputNamesUTF8Encoded.forEach(wasm._OrtFree); this.inputNamesUTF8Encoded = []; } if (this.outputNamesUTF8Encoded) { - this.outputNamesUTF8Encoded.forEach(str => wasm._OrtFree(str)); + this.outputNamesUTF8Encoded.forEach(wasm._OrtFree); this.outputNamesUTF8Encoded = []; } if (this.sessionHandle) { @@ -160,9 +323,8 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { } } - async run( - feeds: SessionHandler.FeedsType, fetches: SessionHandler.FetchesType, - _options: InferenceSession.RunOptions): Promise { + async run(feeds: SessionHandler.FeedsType, fetches: SessionHandler.FetchesType, options: InferenceSession.RunOptions): + Promise { const wasm = getInstance(); const inputArray: Tensor[] = []; @@ -196,112 +358,129 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler { const inputCount = inputIndices.length; const outputCount = outputIndices.length; + let runOptionsHandle = 0; + let allocs: number[] = []; + const inputValues: number[] = []; const inputDataOffsets: number[] = []; - // create input tensors - for (let i = 0; i < inputCount; i++) { - const data = inputArray[i].data; - if (Array.isArray(data)) { - // string tensor - throw new TypeError('string tensor is not supported'); - } else { - const dataOffset = wasm._malloc(data.byteLength); - inputDataOffsets.push(dataOffset); - wasm.HEAPU8.set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), dataOffset); - const dims = inputArray[i].dims; - - const stack = wasm.stackSave(); - const dimsOffset = wasm.stackAlloc(4 * dims.length); - try { - let dimIndex = dimsOffset / 4; - dims.forEach(d => wasm.HEAP32[dimIndex++] = d); - const tensor = wasm._OrtCreateTensor( - tensorDataTypeStringToEnum(inputArray[i].type), dataOffset, data.byteLength, dimsOffset, dims.length); - inputValues.push(tensor); - } finally { - wasm.stackRestore(stack); - } - } - } - - const beforeRunStack = wasm.stackSave(); - const inputValuesOffset = wasm.stackAlloc(inputCount * 4); - const inputNamesOffset = wasm.stackAlloc(inputCount * 4); - const outputValuesOffset = wasm.stackAlloc(outputCount * 4); - const outputNamesOffset = wasm.stackAlloc(outputCount * 4); try { - let inputValuesIndex = inputValuesOffset / 4; - let inputNamesIndex = inputNamesOffset / 4; - let outputValuesIndex = outputValuesOffset / 4; - let outputNamesIndex = outputNamesOffset / 4; + [runOptionsHandle, allocs] = setRunOptions(options); + + // create input tensors for (let i = 0; i < inputCount; i++) { - wasm.HEAPU32[inputValuesIndex++] = inputValues[i]; - wasm.HEAPU32[inputNamesIndex++] = this.inputNamesUTF8Encoded[inputIndices[i]]; - } - for (let i = 0; i < outputCount; i++) { - wasm.HEAPU32[outputValuesIndex++] = 0; - wasm.HEAPU32[outputNamesIndex++] = this.outputNamesUTF8Encoded[outputIndices[i]]; - } + const data = inputArray[i].data; + if (Array.isArray(data)) { + // string tensor + throw new TypeError('string tensor is not supported'); + } else { + const dataOffset = wasm._malloc(data.byteLength); + inputDataOffsets.push(dataOffset); + wasm.HEAPU8.set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), dataOffset); - // support RunOptions - const errorCode = wasm._OrtRun( - this.sessionHandle, inputNamesOffset, inputValuesOffset, inputCount, outputNamesOffset, outputCount, - outputValuesOffset); + const dims = inputArray[i].dims; - const output: {[name: string]: Tensor} = {}; - - if (errorCode === 0) { - for (let i = 0; i < outputCount; i++) { - const tensor = wasm.HEAPU32[outputValuesOffset / 4 + i]; - - const beforeGetTensorDataStack = wasm.stackSave(); - // stack allocate 4 pointer value - const tensorDataOffset = wasm.stackAlloc(4 * 4); + const stack = wasm.stackSave(); + const dimsOffset = wasm.stackAlloc(4 * dims.length); try { - wasm._OrtGetTensorData( - tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12); - let tensorDataIndex = tensorDataOffset / 4; - const dataType = wasm.HEAPU32[tensorDataIndex++]; - const dataOffset: number = wasm.HEAPU32[tensorDataIndex++]; - const dimsOffset = wasm.HEAPU32[tensorDataIndex++]; - const dimsLength = wasm.HEAPU32[tensorDataIndex++]; - const dims = []; - for (let i = 0; i < dimsLength; i++) { - dims.push(wasm.HEAPU32[dimsOffset / 4 + i]); - } - wasm._OrtFree(dimsOffset); - - const type = tensorDataTypeEnumToString(dataType); - if (type === 'string') { - // string tensor - throw new TypeError('string tensor is not supported'); - } else { - const typedArray = numericTensorTypeToTypedArray(type); - const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b); - const t = new Tensor(type, new typedArray(size), dims) as TypedTensor>; - new Uint8Array(t.data.buffer, t.data.byteOffset, t.data.byteLength) - .set(wasm.HEAPU8.subarray(dataOffset, dataOffset + t.data.byteLength)); - output[this.outputNames[outputIndices[i]]] = t; + let dimIndex = dimsOffset / 4; + dims.forEach(d => wasm.HEAP32[dimIndex++] = d); + const tensor = wasm._OrtCreateTensor( + tensorDataTypeStringToEnum(inputArray[i].type), dataOffset, data.byteLength, dimsOffset, dims.length); + if (tensor === 0) { + throw new Error('Can\'t create a tensor'); } + inputValues.push(tensor); } finally { - wasm.stackRestore(beforeGetTensorDataStack); + wasm.stackRestore(stack); } - - wasm._OrtReleaseTensor(tensor); } } - inputValues.forEach(t => wasm._OrtReleaseTensor(t)); - inputDataOffsets.forEach(i => wasm._free(i)); + const beforeRunStack = wasm.stackSave(); + const inputValuesOffset = wasm.stackAlloc(inputCount * 4); + const inputNamesOffset = wasm.stackAlloc(inputCount * 4); + const outputValuesOffset = wasm.stackAlloc(outputCount * 4); + const outputNamesOffset = wasm.stackAlloc(outputCount * 4); - if (errorCode === 0) { - return output; - } else { - throw new Error(`failed to call OrtRun(). error code = ${errorCode}.`); + try { + let inputValuesIndex = inputValuesOffset / 4; + let inputNamesIndex = inputNamesOffset / 4; + let outputValuesIndex = outputValuesOffset / 4; + let outputNamesIndex = outputNamesOffset / 4; + for (let i = 0; i < inputCount; i++) { + wasm.HEAPU32[inputValuesIndex++] = inputValues[i]; + wasm.HEAPU32[inputNamesIndex++] = this.inputNamesUTF8Encoded[inputIndices[i]]; + } + for (let i = 0; i < outputCount; i++) { + wasm.HEAPU32[outputValuesIndex++] = 0; + wasm.HEAPU32[outputNamesIndex++] = this.outputNamesUTF8Encoded[outputIndices[i]]; + } + + // support RunOptions + let errorCode = wasm._OrtRun( + this.sessionHandle, inputNamesOffset, inputValuesOffset, inputCount, outputNamesOffset, outputCount, + outputValuesOffset, runOptionsHandle); + + const output: {[name: string]: Tensor} = {}; + + if (errorCode === 0) { + for (let i = 0; i < outputCount; i++) { + const tensor = wasm.HEAPU32[outputValuesOffset / 4 + i]; + + const beforeGetTensorDataStack = wasm.stackSave(); + // stack allocate 4 pointer value + const tensorDataOffset = wasm.stackAlloc(4 * 4); + try { + errorCode = wasm._OrtGetTensorData( + tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12); + if (errorCode !== 0) { + throw new Error(`Can't get a tensor data. error code = ${errorCode}`); + } + let tensorDataIndex = tensorDataOffset / 4; + const dataType = wasm.HEAPU32[tensorDataIndex++]; + const dataOffset: number = wasm.HEAPU32[tensorDataIndex++]; + const dimsOffset = wasm.HEAPU32[tensorDataIndex++]; + const dimsLength = wasm.HEAPU32[tensorDataIndex++]; + const dims = []; + for (let i = 0; i < dimsLength; i++) { + dims.push(wasm.HEAPU32[dimsOffset / 4 + i]); + } + wasm._OrtFree(dimsOffset); + + const type = tensorDataTypeEnumToString(dataType); + if (type === 'string') { + // string tensor + throw new TypeError('string tensor is not supported'); + } else { + const typedArray = numericTensorTypeToTypedArray(type); + const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b); + const t = new Tensor(type, new typedArray(size), dims) as TypedTensor>; + new Uint8Array(t.data.buffer, t.data.byteOffset, t.data.byteLength) + .set(wasm.HEAPU8.subarray(dataOffset, dataOffset + t.data.byteLength)); + output[this.outputNames[outputIndices[i]]] = t; + } + } finally { + wasm.stackRestore(beforeGetTensorDataStack); + wasm._OrtReleaseTensor(tensor); + } + } + } + + if (errorCode === 0) { + return output; + } else { + throw new Error(`failed to call OrtRun(). error code = ${errorCode}.`); + } + } finally { + wasm.stackRestore(beforeRunStack); } } finally { - wasm.stackRestore(beforeRunStack); + inputValues.forEach(wasm._OrtReleaseTensor); + inputDataOffsets.forEach(wasm._free); + + wasm._OrtReleaseRunOptions(runOptionsHandle); + allocs.forEach(wasm._free); } } diff --git a/js/web/script/test-runner-cli.ts b/js/web/script/test-runner-cli.ts index 350747d130..d657e390ac 100644 --- a/js/web/script/test-runner-cli.ts +++ b/js/web/script/test-runner-cli.ts @@ -568,6 +568,9 @@ function saveConfig(config: Test.Config) { if (config.options.wasmFlags && config.options.wasmFlags.numThreads !== undefined) { setOptions += `ort.env.wasm.numThreads = ${JSON.stringify(config.options.wasmFlags.numThreads)};`; } + if (config.options.wasmFlags && config.options.wasmFlags.loggingLevel !== undefined) { + setOptions += `ort.env.wasm.loggingLevel = ${JSON.stringify(config.options.wasmFlags.loggingLevel)};`; + } if (config.options.wasmFlags && config.options.wasmFlags.initTimeout !== undefined) { setOptions += `ort.env.wasm.initTimeout = ${JSON.stringify(config.options.wasmFlags.initTimeout)};`; } diff --git a/onnxruntime/wasm/api.cc b/onnxruntime/wasm/api.cc index ccf94fd0d6..93d7d09f17 100644 --- a/onnxruntime/wasm/api.cc +++ b/onnxruntime/wasm/api.cc @@ -9,60 +9,156 @@ #include namespace { -Ort::Env* g_env; +OrtEnv* g_env; } // namespace -void OrtInit(int numThreads, int logging_level) { -#if defined(__EMSCRIPTEN_PTHREADS__) - OrtThreadingOptions* tp_options; - Ort::ThrowOnError(Ort::GetApi().CreateThreadingOptions(&tp_options)); - Ort::ThrowOnError(Ort::GetApi().SetGlobalIntraOpNumThreads(tp_options, numThreads)); - Ort::ThrowOnError(Ort::GetApi().SetGlobalInterOpNumThreads(tp_options, 1)); - - g_env = new Ort::Env{tp_options, static_cast(logging_level), "Default"}; -#endif - g_env = new Ort::Env{static_cast(logging_level), "Default"}; +OrtErrorCode CheckStatus(OrtStatusPtr status) { + OrtErrorCode error_code = ORT_OK; + if (status) { + std::string error_message = Ort::GetApi().GetErrorMessage(status); + error_code = Ort::GetApi().GetErrorCode(status); + std::cerr << Ort::Exception(std::move(error_message), error_code).what() << std::endl; + Ort::GetApi().ReleaseStatus(status); + } + return error_code; } -Ort::Session* OrtCreateSession(void* data, size_t data_length) { - Ort::SessionOptions session_options; - session_options.SetLogId("onnxruntime"); +#define CHECK_STATUS(ORT_API_NAME, ...) \ + CheckStatus(Ort::GetApi().ORT_API_NAME(__VA_ARGS__)) + +#define RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \ + do { \ + int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \ + if (error_code != ORT_OK) { \ + return error_code; \ + } \ + } while (false) + +// TODO: This macro can be removed when we changed all APIs to return a status code. +#define RETURN_NULLPTR_IF_ERROR(ORT_API_NAME, ...) \ + do { \ + if (CHECK_STATUS(ORT_API_NAME, __VA_ARGS__) != ORT_OK) { \ + return nullptr; \ + } \ + } while (false) + +int OrtInit(int num_threads, int logging_level) { + // Assume that a logging level is check and properly set at JavaScript +#if defined(__EMSCRIPTEN_PTHREADS__) + OrtThreadingOptions* tp_options = nullptr; + RETURN_ERROR_CODE_IF_ERROR(CreateThreadingOptions, &tp_options); + RETURN_ERROR_CODE_IF_ERROR(SetGlobalIntraOpNumThreads, tp_options, num_threads); + RETURN_ERROR_CODE_IF_ERROR(SetGlobalInterOpNumThreads, tp_options, 1); + + return CHECK_STATUS(CreateEnvWithGlobalThreadPools, + static_cast(logging_level), + "Default", + tp_options, + &g_env); +#else + return CHECK_STATUS(CreateEnv, static_cast(logging_level), "Default", &g_env); +#endif +} + +OrtSessionOptions* OrtCreateSessionOptions() { + OrtSessionOptions* session_options = nullptr; + return (CHECK_STATUS(CreateSessionOptions, &session_options) == ORT_OK) ? session_options : nullptr; +} + +void OrtReleaseSessionOptions(OrtSessionOptions* session_options) { + Ort::GetApi().ReleaseSessionOptions(session_options); +} + +int OrtSetSessionGraphOptimizationLevel(OrtSessionOptions* session_options, size_t level) { + // Assume that a graph optimization level is check and properly set at JavaScript + return CHECK_STATUS(SetSessionGraphOptimizationLevel, session_options, static_cast(level)); +} + +int OrtEnableCpuMemArena(OrtSessionOptions* session_options) { + return CHECK_STATUS(EnableCpuMemArena, session_options); +} + +int OrtDisableCpuMemArena(OrtSessionOptions* session_options) { + return CHECK_STATUS(DisableCpuMemArena, session_options); +} + +int OrtEnableMemPattern(OrtSessionOptions* session_options) { + return CHECK_STATUS(EnableMemPattern, session_options); +} + +int OrtDisableMemPattern(OrtSessionOptions* session_options) { + return CHECK_STATUS(DisableMemPattern, session_options); +} + +int OrtSetSessionExecutionMode(OrtSessionOptions* session_options, size_t mode) { + // Assume that an execution mode is check and properly set at JavaScript + return CHECK_STATUS(SetSessionExecutionMode, session_options, static_cast(mode)); +} + +int OrtSetSessionLogId(OrtSessionOptions* session_options, const char* logid) { + return CHECK_STATUS(SetSessionLogId, session_options, logid); +} + +int OrtSetSessionLogSeverityLevel(OrtSessionOptions* session_options, size_t level) { + return CHECK_STATUS(SetSessionLogSeverityLevel, session_options, level); +} + +OrtSession* OrtCreateSession(void* data, size_t data_length, OrtSessionOptions* session_options) { + // OrtSessionOptions must not be nullptr. + if (session_options == nullptr) { + return nullptr; + } #if defined(__EMSCRIPTEN_PTHREADS__) - session_options.DisablePerSessionThreads(); + RETURN_NULLPTR_IF_ERROR(DisablePerSessionThreads, session_options); #else // must disable thread pool when WebAssembly multi-threads support is disabled. - session_options.SetIntraOpNumThreads(1); + RETURN_NULLPTR_IF_ERROR(SetIntraOpNumThreads, session_options, 1); + RETURN_NULLPTR_IF_ERROR(SetSessionExecutionMode, session_options, ORT_SEQUENTIAL); #endif - return new Ort::Session(*g_env, data, data_length, session_options); + OrtSession* session = nullptr; + return (CHECK_STATUS(CreateSessionFromArray, g_env, data, data_length, session_options, &session) == ORT_OK) + ? session : nullptr; } -void OrtReleaseSession(Ort::Session* session) { - delete session; +void OrtReleaseSession(OrtSession* session) { + Ort::GetApi().ReleaseSession(session); } -size_t OrtGetInputCount(Ort::Session* session) { - return session->GetInputCount(); +size_t OrtGetInputCount(OrtSession* session) { + size_t input_count = 0; + return (CHECK_STATUS(SessionGetInputCount, session, &input_count) == ORT_OK) ? input_count : 0; } -size_t OrtGetOutputCount(Ort::Session* session) { - return session->GetOutputCount(); +size_t OrtGetOutputCount(OrtSession* session) { + size_t output_count = 0; + return (CHECK_STATUS(SessionGetOutputCount, session, &output_count) == ORT_OK) ? output_count : 0; } -char* OrtGetInputName(Ort::Session* session, size_t index) { - Ort::AllocatorWithDefaultOptions allocator; - return session->GetInputName(index, allocator); +char* OrtGetInputName(OrtSession* session, size_t index) { + OrtAllocator* allocator = nullptr; + RETURN_NULLPTR_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator); + + char* input_name = nullptr; + return (CHECK_STATUS(SessionGetInputName, session, index, allocator, &input_name) == ORT_OK) + ? input_name : nullptr; } -char* OrtGetOutputName(Ort::Session* session, size_t index) { - Ort::AllocatorWithDefaultOptions allocator; - return session->GetOutputName(index, allocator); +char* OrtGetOutputName(OrtSession* session, size_t index) { + OrtAllocator* allocator = nullptr; + RETURN_NULLPTR_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator); + + char* output_name = nullptr; + return (CHECK_STATUS(SessionGetOutputName, session, index, allocator, &output_name) == ORT_OK) + ? output_name : nullptr; } void OrtFree(void* ptr) { - Ort::AllocatorWithDefaultOptions allocator; - allocator.Free(ptr); + OrtAllocator* allocator = nullptr; + if (CHECK_STATUS(GetAllocatorWithDefaultOptions, &allocator) == ORT_OK) { + allocator->Free(allocator, ptr); + } } OrtValue* OrtCreateTensor(int data_type, void* data, size_t data_length, size_t* dims, size_t dims_length) { @@ -71,47 +167,87 @@ OrtValue* OrtCreateTensor(int data_type, void* data, size_t data_length, size_t* shapes[i] = dims[i]; } - return Ort::Value::CreateTensor(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault), - data, - data_length, - dims_length > 0 ? shapes.data() : nullptr, - dims_length, - static_cast(data_type)) - .release(); + OrtMemoryInfo* memoryInfo = nullptr; + RETURN_NULLPTR_IF_ERROR(CreateCpuMemoryInfo, OrtDeviceAllocator, OrtMemTypeDefault, &memoryInfo); + + OrtValue* value = nullptr; + int error_code = CHECK_STATUS(CreateTensorWithDataAsOrtValue, memoryInfo, data, data_length, + dims_length > 0 ? shapes.data() : nullptr, dims_length, + static_cast(data_type), &value); + + Ort::GetApi().ReleaseMemoryInfo(memoryInfo); + return (error_code == ORT_OK) ? value : nullptr; } -void OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dims, size_t* dims_length) { - Ort::Value v{tensor}; - auto info = v.GetTensorTypeAndShapeInfo(); - size_t dims_len = info.GetDimensionsCount(); - Ort::AllocatorWithDefaultOptions allocator; - size_t* p_dims = reinterpret_cast(allocator.Alloc(sizeof(size_t) * dims_len)); - *data = v.GetTensorMutableData(); - *data_type = info.GetElementType(); +int OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dims, size_t* dims_length) { +#define RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \ + do { \ + int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \ + if (error_code != ORT_OK) { \ + if (info != nullptr) { \ + Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); \ + } \ + if (allocator != nullptr && p_dims != nullptr) { \ + allocator->Free(allocator, p_dims); \ + } \ + return error_code; \ + } \ + } while (false) + + OrtTensorTypeAndShapeInfo* info = nullptr; + OrtAllocator* allocator = nullptr; + size_t* p_dims = nullptr; + + RETURN_ERROR_CODE_IF_ERROR(GetTensorTypeAndShape, tensor, &info); + + size_t dims_len = 0; + RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetDimensionsCount, info, &dims_len); + + RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator); + p_dims = reinterpret_cast(allocator->Alloc(allocator, sizeof(size_t) * dims_len)); + + RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorMutableData, tensor, data); + + ONNXTensorElementDataType type; + RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorElementType, info, &type); + *data_type = static_cast(type); + *dims_length = dims_len; - auto shape = info.GetShape(); + std::vector shape(dims_len, 0); + RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetDimensions, info, shape.data(), shape.size()); for (size_t i = 0; i < dims_len; i++) { p_dims[i] = static_cast(shape[i]); } *dims = p_dims; - v.release(); + + Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); + return ORT_OK; } void OrtReleaseTensor(OrtValue* tensor) { - Ort::OrtRelease(tensor); + Ort::GetApi().ReleaseValue(tensor); } -int OrtRun(Ort::Session* session, - const char** input_names, const ort_tensor_handle_t* inputs, size_t input_count, - const char** output_names, size_t output_count, ort_tensor_handle_t* outputs) { - OrtStatusPtr status = Ort::GetApi().Run(*session, Ort::RunOptions{nullptr}, input_names, inputs, input_count, output_names, output_count, outputs); - OrtErrorCode error_code = ORT_OK; - if (status) { - std::string error_message = Ort::GetApi().GetErrorMessage(status); - error_code = Ort::GetApi().GetErrorCode(status); - std::cerr << Ort::Exception(std::move(error_message), error_code).what() - << std::endl; - Ort::GetApi().ReleaseStatus(status); - } - return error_code; +OrtRunOptions* OrtCreateRunOptions() { + OrtRunOptions* run_options = nullptr; + return (CHECK_STATUS(CreateRunOptions, &run_options) == ORT_OK) ? run_options : nullptr; +} + +void OrtReleaseRunOptions(OrtRunOptions* run_options) { + Ort::GetApi().ReleaseRunOptions(run_options); +} + +int OrtRunOptionsSetRunLogSeverityLevel(OrtRunOptions* run_options, size_t level) { + return CHECK_STATUS(RunOptionsSetRunLogSeverityLevel, run_options, level); +} + +int OrtRunOptionsSetRunTag(OrtRunOptions* run_options, const char* tag) { + return CHECK_STATUS(RunOptionsSetRunTag, run_options, tag); +} + +int OrtRun(OrtSession* session, + const char** input_names, const ort_tensor_handle_t* inputs, size_t input_count, + const char** output_names, size_t output_count, ort_tensor_handle_t* outputs, + OrtRunOptions* run_options) { + return CHECK_STATUS(Run, session, run_options, input_names, inputs, input_count, output_names, output_count, outputs); } diff --git a/onnxruntime/wasm/api.h b/onnxruntime/wasm/api.h index 80e18b2dc2..4c54cd7d52 100644 --- a/onnxruntime/wasm/api.h +++ b/onnxruntime/wasm/api.h @@ -12,10 +12,14 @@ #include -namespace Ort { -struct Session; -} -using ort_session_handle_t = Ort::Session*; +struct OrtSession; +using ort_session_handle_t = OrtSession*; + +struct OrtSessionOptions; +using ort_session_options_handle_t = OrtSessionOptions*; + +struct OrtRunOptions; +using ort_run_options_handle_t = OrtRunOptions*; struct OrtValue; using ort_tensor_handle_t = OrtValue*; @@ -27,7 +31,58 @@ extern "C" { * @param numThreads number of total threads to use. * @param logging_level default logging level. */ -void EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level); +int EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level); + +/** + * create an instance of ORT session options. + * @returns a pointer to a session option handle and must be freed by calling OrtReleaseSessionOptions(). + */ +ort_session_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSessionOptions(); + +/** + * release the specified ORT session options. + */ +void EMSCRIPTEN_KEEPALIVE OrtReleaseSessionOptions(ort_session_options_handle_t session_options); + +/** + * set an optimization level for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtSetSessionGraphOptimizationLevel(ort_session_options_handle_t session_options, size_t level); + +/** + * enable CPU memory arena for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtEnableCpuMemArena(ort_session_options_handle_t session_options); + +/** + * disable CPU memory arena for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtDisableCpuMemArena(ort_session_options_handle_t session_options); + +/** + * enable memory pattern for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtEnableMemPattern(ort_session_options_handle_t session_options); + +/** + * disable memory pattern for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtDisableMemPattern(ort_session_options_handle_t session_options); + +/** + * set an execution mode for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtSetSessionExecutionMode(ort_session_options_handle_t session_options, size_t mode); + +/** + * set a log ID for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtSetSessionLogId(ort_session_options_handle_t session_options, const char* logid); + +/** + * set a log severity level for session. + */ +int EMSCRIPTEN_KEEPALIVE OrtSetSessionLogSeverityLevel(ort_session_options_handle_t session_options, size_t level); /** * create an instance of ORT session. @@ -35,7 +90,9 @@ void EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level); * @param data_length the size of the buffer in bytes. * @returns a handle of the ORT session. */ -ort_session_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSession(void* data, size_t data_length); +ort_session_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSession(void* data, + size_t data_length, + ort_session_options_handle_t session_options); /** * release the specified ORT session. @@ -88,13 +145,34 @@ ort_tensor_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateTensor(int data_type, void* da * @param dims_length [out] specify the memory to write dims length * @remarks a temporary buffer 'dims' is allocated during the call. Caller must release the buffer after use by calling OrtFree(). */ -void EMSCRIPTEN_KEEPALIVE OrtGetTensorData(ort_tensor_handle_t tensor, int* data_type, void** data, size_t** dims, size_t* dims_length); +int EMSCRIPTEN_KEEPALIVE OrtGetTensorData(ort_tensor_handle_t tensor, int* data_type, void** data, size_t** dims, size_t* dims_length); /** * release the specified tensor. */ void EMSCRIPTEN_KEEPALIVE OrtReleaseTensor(ort_tensor_handle_t tensor); +/** + * create an instance of ORT run options. + * @returns a pointer to a run option handle and must be freed by calling OrtReleaseRunOptions(). + */ +ort_run_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateRunOptions(); + +/** + * release the specified ORT run options. + */ +void EMSCRIPTEN_KEEPALIVE OrtReleaseRunOptions(ort_run_options_handle_t run_options); + +/** + * set log severity level for run. + */ +int EMSCRIPTEN_KEEPALIVE OrtRunOptionsSetRunLogSeverityLevel(ort_run_options_handle_t run_options, size_t level); + +/** + * set a tag for the Run() calls using this. + */ +int EMSCRIPTEN_KEEPALIVE OrtRunOptionsSetRunTag(ort_run_options_handle_t run_options, const char* tag); + /** * inference the model. * @param session handle of the specified session @@ -106,5 +184,6 @@ int EMSCRIPTEN_KEEPALIVE OrtRun(ort_session_handle_t session, size_t input_count, const char** output_names, size_t output_count, - ort_tensor_handle_t* outputs); + ort_tensor_handle_t* outputs, + ort_run_options_handle_t run_options); };