diff --git a/js/web/lib/wasm/binding/ort-wasm.d.ts b/js/web/lib/wasm/binding/ort-wasm.d.ts index 2e51d3257e..4f1662199a 100644 --- a/js/web/lib/wasm/binding/ort-wasm.d.ts +++ b/js/web/lib/wasm/binding/ort-wasm.d.ts @@ -26,10 +26,11 @@ export interface OrtWasmModule extends EmscriptenModule { // #region ORT APIs _OrtInit(numThreads: number, loggingLevel: number): number; + _OrtGetLastError(errorCodeOffset: number, errorMessageOffset: number): void; + _OrtCreateSession(dataOffset: number, dataLength: number, sessionOptionsHandle: number): number; _OrtReleaseSession(sessionHandle: number): void; - _OrtGetInputCount(sessionHandle: number): number; - _OrtGetOutputCount(sessionHandle: number): number; + _OrtGetInputOutputCount(sessionHandle: number, inputCountOffset: number, outputCountOffset: number): number; _OrtGetInputName(sessionHandle: number, index: number): number; _OrtGetOutputName(sessionHandle: number, index: number): number; diff --git a/js/web/lib/wasm/run-options.ts b/js/web/lib/wasm/run-options.ts index 260a622a41..8fe2300034 100644 --- a/js/web/lib/wasm/run-options.ts +++ b/js/web/lib/wasm/run-options.ts @@ -3,9 +3,8 @@ import {InferenceSession} from 'onnxruntime-common'; -import {iterateExtraOptions} from './options-utils'; -import {allocWasmString} from './string-utils'; import {getInstance} from './wasm-factory'; +import {allocWasmString, checkLastError, iterateExtraOptions} from './wasm-utils'; export const setRunOptions = (options: InferenceSession.RunOptions): [number, number[]] => { const wasm = getInstance(); @@ -41,7 +40,7 @@ export const setRunOptions = (options: InferenceSession.RunOptions): [number, nu runOptionsHandle = wasm._OrtCreateRunOptions( runOptions.logSeverityLevel!, runOptions.logVerbosityLevel!, !!runOptions.terminate!, tagDataOffset); if (runOptionsHandle === 0) { - throw new Error('Can\'t create run options'); + checkLastError('Can\'t create run options.'); } if (options?.extra !== undefined) { @@ -50,7 +49,7 @@ export const setRunOptions = (options: InferenceSession.RunOptions): [number, nu const valueDataOffset = allocWasmString(value, allocs); if (wasm._OrtAddRunConfigEntry(runOptionsHandle, keyDataOffset, valueDataOffset) !== 0) { - throw new Error(`Can't set a run config entry: ${key} - ${value}`); + checkLastError(`Can't set a run config entry: ${key} - ${value}.`); } }); } @@ -60,7 +59,7 @@ export const setRunOptions = (options: InferenceSession.RunOptions): [number, nu if (runOptionsHandle !== 0) { wasm._OrtReleaseRunOptions(runOptionsHandle); } - allocs.forEach(wasm._free); + allocs.forEach(alloc => wasm._free(alloc)); throw e; } }; diff --git a/js/web/lib/wasm/session-options.ts b/js/web/lib/wasm/session-options.ts index c27d2f1e17..38caa9076e 100644 --- a/js/web/lib/wasm/session-options.ts +++ b/js/web/lib/wasm/session-options.ts @@ -3,9 +3,8 @@ import {InferenceSession} from 'onnxruntime-common'; -import {iterateExtraOptions} from './options-utils'; -import {allocWasmString} from './string-utils'; import {getInstance} from './wasm-factory'; +import {allocWasmString, checkLastError, iterateExtraOptions} from './wasm-utils'; const getGraphOptimzationLevel = (graphOptimizationLevel: string|unknown): number => { switch (graphOptimizationLevel) { @@ -73,7 +72,7 @@ const setExecutionProviders = const valueDataOffset = allocWasmString(webnnOptions.deviceType, allocs); if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !== 0) { - throw new Error(`Can't set a session config entry: 'deviceType' - ${webnnOptions.deviceType}`); + checkLastError(`Can't set a session config entry: 'deviceType' - ${webnnOptions.deviceType}.`); } } if (webnnOptions?.powerPreference) { @@ -81,8 +80,8 @@ const setExecutionProviders = const valueDataOffset = allocWasmString(webnnOptions.powerPreference, allocs); if (getInstance()._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !== 0) { - throw new Error( - `Can't set a session config entry: 'powerPreference' - ${webnnOptions.powerPreference}`); + checkLastError( + `Can't set a session config entry: 'powerPreference' - ${webnnOptions.powerPreference}.`); } } } @@ -94,12 +93,12 @@ const setExecutionProviders = case 'cpu': continue; default: - throw new Error(`not supported EP: ${epName}`); + throw new Error(`not supported execution provider: ${epName}`); } const epNameDataOffset = allocWasmString(epName, allocs); if (getInstance()._OrtAppendExecutionProvider(sessionOptionsHandle, epNameDataOffset) !== 0) { - throw new Error(`Can't append execution provider: ${epName}`); + checkLastError(`Can't append execution provider: ${epName}.`); } } }; @@ -137,7 +136,7 @@ export const setSessionOptions = (options?: InferenceSession.SessionOptions): [n !!sessionOptions.enableProfiling, 0, logIdDataOffset, logSeverityLevel, logVerbosityLevel, optimizedModelFilePathOffset); if (sessionOptionsHandle === 0) { - throw new Error('Can\'t create session options'); + checkLastError('Can\'t create session options.'); } if (sessionOptions.executionProviders) { @@ -150,7 +149,7 @@ export const setSessionOptions = (options?: InferenceSession.SessionOptions): [n const valueDataOffset = allocWasmString(value, allocs); if (wasm._OrtAddSessionConfigEntry(sessionOptionsHandle, keyDataOffset, valueDataOffset) !== 0) { - throw new Error(`Can't set a session config entry: ${key} - ${value}`); + checkLastError(`Can't set a session config entry: ${key} - ${value}.`); } }); } @@ -160,7 +159,7 @@ export const setSessionOptions = (options?: InferenceSession.SessionOptions): [n if (sessionOptionsHandle !== 0) { wasm._OrtReleaseSessionOptions(sessionOptionsHandle); } - allocs.forEach(wasm._free); + allocs.forEach(alloc => wasm._free(alloc)); throw e; } }; diff --git a/js/web/lib/wasm/string-utils.ts b/js/web/lib/wasm/string-utils.ts deleted file mode 100644 index d7ea4a4626..0000000000 --- a/js/web/lib/wasm/string-utils.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import {getInstance} from './wasm-factory'; - -export const allocWasmString = (data: string, allocs: number[]): number => { - const wasm = getInstance(); - - const dataLength = wasm.lengthBytesUTF8(data) + 1; - const dataOffset = wasm._malloc(dataLength); - wasm.stringToUTF8(data, dataOffset, dataLength); - allocs.push(dataOffset); - - return dataOffset; -}; diff --git a/js/web/lib/wasm/wasm-common.ts b/js/web/lib/wasm/wasm-common.ts index d0d66326c7..389773f3e8 100644 --- a/js/web/lib/wasm/wasm-common.ts +++ b/js/web/lib/wasm/wasm-common.ts @@ -3,6 +3,8 @@ import {Tensor} from 'onnxruntime-common'; +// This file includes common definitions. They do NOT have dependency on the WebAssembly instance. + /** * Copied from ONNX definition. Use this to drop dependency 'onnx_proto' to decrease compiled .js file size. */ diff --git a/js/web/lib/wasm/wasm-core-impl.ts b/js/web/lib/wasm/wasm-core-impl.ts index 3dadedffc3..2b339afc46 100644 --- a/js/web/lib/wasm/wasm-core-impl.ts +++ b/js/web/lib/wasm/wasm-core-impl.ts @@ -7,19 +7,39 @@ import {init as initJsep} from './jsep/init'; import {SerializableModeldata, SerializableSessionMetadata, SerializableTensor} from './proxy-messages'; import {setRunOptions} from './run-options'; import {setSessionOptions} from './session-options'; -import {allocWasmString} from './string-utils'; import {logLevelStringToEnum, tensorDataTypeEnumToString, tensorDataTypeStringToEnum, tensorTypeToTypedArrayConstructor} from './wasm-common'; import {getInstance} from './wasm-factory'; +import {allocWasmString, checkLastError} from './wasm-utils'; + +/** + * get the input/output count of the session. + * @param sessionHandle the handle representing the session. should be non-zero. + * @returns a tuple including 2 numbers, representing the input count and output count. + */ +const getSessionInputOutputCount = (sessionHandle: number): [number, number] => { + const wasm = getInstance(); + const stack = wasm.stackSave(); + try { + const dataOffset = wasm.stackAlloc(8); + const errorCode = wasm._OrtGetInputOutputCount(sessionHandle, dataOffset, dataOffset + 4); + if (errorCode !== 0) { + checkLastError('Can\'t get session input/output count.'); + } + return [wasm.HEAP32[dataOffset / 4], wasm.HEAP32[dataOffset / 4 + 1]]; + } finally { + wasm.stackRestore(stack); + } +}; /** * initialize ORT environment. * @param numThreads SetGlobalIntraOpNumThreads(numThreads) * @param loggingLevel CreateEnv(static_cast(logging_level)) */ -const initOrt = async(numThreads: number, loggingLevel: number): Promise => { +const initOrt = (numThreads: number, loggingLevel: number): void => { const errorCode = getInstance()._OrtInit(numThreads, loggingLevel); if (errorCode !== 0) { - throw new Error(`Can't initialize onnxruntime. error code = ${errorCode}`); + checkLastError('Can\'t initialize onnxruntime.'); } }; @@ -29,7 +49,7 @@ const initOrt = async(numThreads: number, loggingLevel: number): Promise = */ export const initRuntime = async(env: Env): Promise => { // init ORT - await initOrt(env.wasm.numThreads!, logLevelStringToEnum(env.logLevel)); + initOrt(env.wasm.numThreads!, logLevelStringToEnum(env.logLevel)); // init JSEP if available await initJsep(getInstance(), env); @@ -43,16 +63,25 @@ type SessionMetadata = [number, number[], number[]]; const activeSessions = new Map(); /** - * create an instance of InferenceSession. - * @returns the metadata of InferenceSession. 0-value handle for failure. + * allocate the memory and memcpy the model bytes, preparing for creating an instance of InferenceSession. + * @returns a 2-elements tuple - the pointer and size of the allocated buffer */ export const createSessionAllocate = (model: Uint8Array): [number, number] => { const wasm = getInstance(); const modelDataOffset = wasm._malloc(model.byteLength); + if (modelDataOffset === 0) { + throw new Error(`Can't create a session. failed to allocate a buffer of size ${model.byteLength}.`); + } wasm.HEAPU8.set(model, modelDataOffset); return [modelDataOffset, model.byteLength]; }; +/** + * create an inference session using the prepared buffer containing the model data. + * @param modelData a 2-elements tuple containing the pointer and size of the model data buffer. + * @param options an optional session options object. + * @returns a 3-elements tuple containing [session handle, input names, output names] + */ export const createSessionFinalize = (modelData: SerializableModeldata, options?: InferenceSession.SessionOptions): SerializableSessionMetadata => { const wasm = getInstance(); @@ -60,48 +89,55 @@ export const createSessionFinalize = let sessionHandle = 0; let sessionOptionsHandle = 0; let allocs: number[] = []; + const inputNamesUTF8Encoded = []; + const outputNamesUTF8Encoded = []; try { [sessionOptionsHandle, allocs] = setSessionOptions(options); sessionHandle = wasm._OrtCreateSession(modelData[0], modelData[1], sessionOptionsHandle); if (sessionHandle === 0) { - throw new Error('Can\'t create a session'); + checkLastError('Can\'t create a session.'); } + + const [inputCount, outputCount] = getSessionInputOutputCount(sessionHandle); + + const inputNames = []; + const outputNames = []; + for (let i = 0; i < inputCount; i++) { + const name = wasm._OrtGetInputName(sessionHandle, i); + if (name === 0) { + checkLastError('Can\'t get an input name.'); + } + inputNamesUTF8Encoded.push(name); + inputNames.push(wasm.UTF8ToString(name)); + } + for (let i = 0; i < outputCount; i++) { + const name = wasm._OrtGetOutputName(sessionHandle, i); + if (name === 0) { + checkLastError('Can\'t get an output name.'); + } + outputNamesUTF8Encoded.push(name); + outputNames.push(wasm.UTF8ToString(name)); + } + + activeSessions.set(sessionHandle, [sessionHandle, inputNamesUTF8Encoded, outputNamesUTF8Encoded]); + return [sessionHandle, inputNames, outputNames]; + } catch (e) { + inputNamesUTF8Encoded.forEach(buf => wasm._OrtFree(buf)); + outputNamesUTF8Encoded.forEach(buf => wasm._OrtFree(buf)); + + if (sessionHandle !== 0) { + wasm._OrtReleaseSession(sessionHandle); + } + throw e; } finally { wasm._free(modelData[0]); if (sessionOptionsHandle !== 0) { wasm._OrtReleaseSessionOptions(sessionOptionsHandle); } - allocs.forEach(wasm._free); + allocs.forEach(alloc => wasm._free(alloc)); } - - const inputCount = wasm._OrtGetInputCount(sessionHandle); - const outputCount = wasm._OrtGetOutputCount(sessionHandle); - - const inputNames = []; - const inputNamesUTF8Encoded = []; - const outputNames = []; - const outputNamesUTF8Encoded = []; - for (let i = 0; i < inputCount; i++) { - const name = wasm._OrtGetInputName(sessionHandle, i); - if (name === 0) { - throw new Error('Can\'t get an input name'); - } - inputNamesUTF8Encoded.push(name); - inputNames.push(wasm.UTF8ToString(name)); - } - for (let i = 0; i < outputCount; i++) { - const name = wasm._OrtGetOutputName(sessionHandle, i); - if (name === 0) { - throw new Error('Can\'t get an output name'); - } - outputNamesUTF8Encoded.push(name); - outputNames.push(wasm.UTF8ToString(name)); - } - - activeSessions.set(sessionHandle, [sessionHandle, inputNamesUTF8Encoded, outputNamesUTF8Encoded]); - return [sessionHandle, inputNames, outputNames]; }; @@ -119,14 +155,12 @@ export const releaseSession = (sessionId: number): void => { const wasm = getInstance(); const session = activeSessions.get(sessionId); if (!session) { - throw new Error('invalid session id'); + throw new Error(`cannot release session. invalid session id: ${sessionId}`); } - const sessionHandle = session[0]; - const inputNamesUTF8Encoded = session[1]; - const outputNamesUTF8Encoded = session[2]; + const [sessionHandle, inputNamesUTF8Encoded, outputNamesUTF8Encoded] = session; - inputNamesUTF8Encoded.forEach(wasm._OrtFree); - outputNamesUTF8Encoded.forEach(wasm._OrtFree); + inputNamesUTF8Encoded.forEach(buf => wasm._OrtFree(buf)); + outputNamesUTF8Encoded.forEach(buf => wasm._OrtFree(buf)); wasm._OrtReleaseSession(sessionHandle); activeSessions.delete(sessionId); }; @@ -140,11 +174,9 @@ export const run = async( const wasm = getInstance(); const session = activeSessions.get(sessionId); if (!session) { - throw new Error('invalid session id'); + throw new Error(`cannot run inference. invalid session id: ${sessionId}`); } - const sessionHandle = session[0]; - const inputNamesUTF8Encoded = session[1]; - const outputNamesUTF8Encoded = session[2]; + const [sessionHandle, inputNamesUTF8Encoded, outputNamesUTF8Encoded] = session; const inputCount = inputIndices.length; const outputCount = outputIndices.length; @@ -194,7 +226,7 @@ export const run = async( const tensor = wasm._OrtCreateTensor( tensorDataTypeStringToEnum(dataType), dataOffset, dataByteLength, dimsOffset, dims.length); if (tensor === 0) { - throw new Error('Can\'t create a tensor'); + checkLastError(`Can't create tensor for input[${i}].`); } inputValues.push(tensor); } finally { @@ -235,74 +267,74 @@ export const run = async( const output: SerializableTensor[] = []; - if (errorCode === 0) { - for (let i = 0; i < outputCount; i++) { - const tensor = wasm.HEAPU32[outputValuesOffset / 4 + i]; + if (errorCode !== 0) { + checkLastError('failed to call OrtRun().'); + } - const beforeGetTensorDataStack = wasm.stackSave(); - // stack allocate 4 pointer value - const tensorDataOffset = wasm.stackAlloc(4 * 4); + for (let i = 0; i < outputCount; i++) { + const tensor = wasm.HEAPU32[outputValuesOffset / 4 + i]; - let type: Tensor.Type|undefined, dataOffset = 0; - try { - errorCode = wasm._OrtGetTensorData( - tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12); - if (errorCode !== 0) { - throw new Error(`Can't access output tensor data. error code = ${errorCode}`); - } - let tensorDataIndex = tensorDataOffset / 4; - const dataType = wasm.HEAPU32[tensorDataIndex++]; - dataOffset = wasm.HEAPU32[tensorDataIndex++]; - const dimsOffset = wasm.HEAPU32[tensorDataIndex++]; - const dimsLength = wasm.HEAPU32[tensorDataIndex++]; - const dims = []; - for (let i = 0; i < dimsLength; i++) { - dims.push(wasm.HEAPU32[dimsOffset / 4 + i]); - } - wasm._OrtFree(dimsOffset); + const beforeGetTensorDataStack = wasm.stackSave(); + // stack allocate 4 pointer value + const tensorDataOffset = wasm.stackAlloc(4 * 4); - const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b); - type = tensorDataTypeEnumToString(dataType); - if (type === 'string') { - const stringData: string[] = []; - let dataIndex = dataOffset / 4; - for (let i = 0; i < size; i++) { - const offset = wasm.HEAPU32[dataIndex++]; - const maxBytesToRead = i === size - 1 ? undefined : wasm.HEAPU32[dataIndex] - offset; - stringData.push(wasm.UTF8ToString(offset, maxBytesToRead)); - } - output.push([type, dims, stringData]); - } else { - const typedArrayConstructor = tensorTypeToTypedArrayConstructor(type); - const data = new typedArrayConstructor(size); - new Uint8Array(data.buffer, data.byteOffset, data.byteLength) - .set(wasm.HEAPU8.subarray(dataOffset, dataOffset + data.byteLength)); - output.push([type, dims, data]); - } - } finally { - wasm.stackRestore(beforeGetTensorDataStack); - if (type === 'string' && dataOffset) { - wasm._free(dataOffset); - } - wasm._OrtReleaseTensor(tensor); + let type: Tensor.Type|undefined, dataOffset = 0; + try { + errorCode = wasm._OrtGetTensorData( + tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12); + if (errorCode !== 0) { + checkLastError(`Can't access output tensor data on index ${i}.`); } + let tensorDataIndex = tensorDataOffset / 4; + const dataType = wasm.HEAPU32[tensorDataIndex++]; + dataOffset = wasm.HEAPU32[tensorDataIndex++]; + const dimsOffset = wasm.HEAPU32[tensorDataIndex++]; + const dimsLength = wasm.HEAPU32[tensorDataIndex++]; + const dims = []; + for (let i = 0; i < dimsLength; i++) { + dims.push(wasm.HEAPU32[dimsOffset / 4 + i]); + } + wasm._OrtFree(dimsOffset); + + const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b); + type = tensorDataTypeEnumToString(dataType); + if (type === 'string') { + const stringData: string[] = []; + let dataIndex = dataOffset / 4; + for (let i = 0; i < size; i++) { + const offset = wasm.HEAPU32[dataIndex++]; + const maxBytesToRead = i === size - 1 ? undefined : wasm.HEAPU32[dataIndex] - offset; + stringData.push(wasm.UTF8ToString(offset, maxBytesToRead)); + } + output.push([type, dims, stringData]); + } else { + const typedArrayConstructor = tensorTypeToTypedArrayConstructor(type); + const data = new typedArrayConstructor(size); + new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + .set(wasm.HEAPU8.subarray(dataOffset, dataOffset + data.byteLength)); + output.push([type, dims, data]); + } + } finally { + wasm.stackRestore(beforeGetTensorDataStack); + if (type === 'string' && dataOffset) { + wasm._free(dataOffset); + } + wasm._OrtReleaseTensor(tensor); } } - if (errorCode === 0) { - return output; - } else { - throw new Error(`failed to call OrtRun(). error code = ${errorCode}.`); - } + return output; } finally { wasm.stackRestore(beforeRunStack); } } finally { - inputValues.forEach(wasm._OrtReleaseTensor); - inputAllocs.forEach(wasm._free); + inputValues.forEach(v => wasm._OrtReleaseTensor(v)); + inputAllocs.forEach(p => wasm._free(p)); - wasm._OrtReleaseRunOptions(runOptionsHandle); - runOptionsAllocs.forEach(wasm._free); + if (runOptionsHandle !== 0) { + wasm._OrtReleaseRunOptions(runOptionsHandle); + } + runOptionsAllocs.forEach(p => wasm._free(p)); } }; @@ -320,7 +352,7 @@ export const endProfiling = (sessionId: number): void => { // profile file name is not used yet, but it must be freed. const profileFileName = wasm._OrtEndProfiling(sessionHandle); if (profileFileName === 0) { - throw new Error('Can\'t get an profile file name'); + checkLastError('Can\'t get an profile file name.'); } wasm._OrtFree(profileFileName); }; diff --git a/js/web/lib/wasm/options-utils.ts b/js/web/lib/wasm/wasm-utils.ts similarity index 51% rename from js/web/lib/wasm/options-utils.ts rename to js/web/lib/wasm/wasm-utils.ts index 21951eff7f..37762b353f 100644 --- a/js/web/lib/wasm/options-utils.ts +++ b/js/web/lib/wasm/wasm-utils.ts @@ -1,6 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +import {getInstance} from './wasm-factory'; + +export const allocWasmString = (data: string, allocs: number[]): number => { + const wasm = getInstance(); + + const dataLength = wasm.lengthBytesUTF8(data) + 1; + const dataOffset = wasm._malloc(dataLength); + wasm.stringToUTF8(data, dataOffset, dataLength); + allocs.push(dataOffset); + + return dataOffset; +}; + interface ExtraOptionsHandler { (name: string, value: string): void; } @@ -29,3 +42,23 @@ export const iterateExtraOptions = } }); }; + +/** + * check web assembly API's last error and throw error if any error occurred. + * @param message a message used when an error occurred. + */ +export const checkLastError = (message: string): void => { + const wasm = getInstance(); + + const stack = wasm.stackSave(); + try { + const paramsOffset = wasm.stackAlloc(8); + wasm._OrtGetLastError(paramsOffset, paramsOffset + 4); + const errorCode = wasm.HEAP32[paramsOffset / 4]; + const errorMessagePointer = wasm.HEAPU32[paramsOffset / 4 + 1]; + const errorMessage = errorMessagePointer ? wasm.UTF8ToString(errorMessagePointer) : ''; + throw new Error(`${message} ERROR_CODE: ${errorCode}, ERROR_MESSAGE: ${errorMessage}`); + } finally { + wasm.stackRestore(stack); + } +}; diff --git a/onnxruntime/wasm/api.cc b/onnxruntime/wasm/api.cc index 5dd0bd2335..a24fb81d49 100644 --- a/onnxruntime/wasm/api.cc +++ b/onnxruntime/wasm/api.cc @@ -10,17 +10,24 @@ namespace { OrtEnv* g_env; +OrtErrorCode g_last_error_code; +std::string g_last_error_message; } // namespace +static_assert(sizeof(const char*) == sizeof(size_t), "size of a pointer and a size_t value should be the same."); +static_assert(sizeof(size_t) == 4, "size of size_t should be 4 in this build (wasm32)."); + OrtErrorCode CheckStatus(OrtStatusPtr status) { - OrtErrorCode error_code = ORT_OK; if (status) { std::string error_message = Ort::GetApi().GetErrorMessage(status); - error_code = Ort::GetApi().GetErrorCode(status); - std::cerr << Ort::Exception(std::move(error_message), error_code).what() << std::endl; + g_last_error_code = Ort::GetApi().GetErrorCode(status); + g_last_error_message = Ort::Exception(std::move(error_message), g_last_error_code).what(); Ort::GetApi().ReleaseStatus(status); + } else { + g_last_error_code = ORT_OK; + g_last_error_message.clear(); } - return error_code; + return g_last_error_code; } #define CHECK_STATUS(ORT_API_NAME, ...) \ @@ -34,7 +41,6 @@ OrtErrorCode CheckStatus(OrtStatusPtr status) { } \ } while (false) -// TODO: This macro can be removed when we changed all APIs to return a status code. #define RETURN_NULLPTR_IF_ERROR(ORT_API_NAME, ...) \ do { \ if (CHECK_STATUS(ORT_API_NAME, __VA_ARGS__) != ORT_OK) { \ @@ -42,6 +48,21 @@ OrtErrorCode CheckStatus(OrtStatusPtr status) { } \ } while (false) +// use auto release macros to make sure resources get released on function return. + +// create a unique_ptr wrapper for auto release +#define REGISTER_AUTO_RELEASE(T, var, release_t, release_func) \ + std::unique_ptr auto_release_##var { var, release_func } +// register auto release for handle of Ort API resources +#define REGISTER_AUTO_RELEASE_HANDLE(T, var) \ + REGISTER_AUTO_RELEASE(Ort##T, var, void (*)(Ort##T*), [](Ort##T* p) { Ort::GetApi().Release##T(p); }) +// register auto release for Ort allocated buffers +#define REGISTER_AUTO_RELEASE_BUFFER(T, var, allocator) \ + auto auto_release_##var##_deleter = [allocator](T* p) { allocator->Free(allocator, p); }; \ + REGISTER_AUTO_RELEASE(T, var, decltype(auto_release_##var##_deleter), auto_release_##var##_deleter) +// unregister the auto release wrapper +#define UNREGISTER_AUTO_RELEASE(var) auto_release_##var.release() + int OrtInit(int num_threads, int logging_level) { // Assume that a logging level is check and properly set at JavaScript #if defined(__EMSCRIPTEN_PTHREADS__) @@ -60,6 +81,11 @@ int OrtInit(int num_threads, int logging_level) { #endif } +void OrtGetLastError(int* error_code, const char** error_message) { + *error_code = g_last_error_code; + *error_message = g_last_error_message.empty() ? nullptr : g_last_error_message.c_str(); +} + OrtSessionOptions* OrtCreateSessionOptions(size_t graph_optimization_level, bool enable_cpu_mem_arena, bool enable_mem_pattern, @@ -72,6 +98,7 @@ OrtSessionOptions* OrtCreateSessionOptions(size_t graph_optimization_level, const char* optimized_model_filepath) { OrtSessionOptions* session_options = nullptr; RETURN_NULLPTR_IF_ERROR(CreateSessionOptions, &session_options); + REGISTER_AUTO_RELEASE_HANDLE(SessionOptions, session_options); if (optimized_model_filepath) { RETURN_NULLPTR_IF_ERROR(SetOptimizedModelFilePath, session_options, optimized_model_filepath); @@ -118,7 +145,7 @@ OrtSessionOptions* OrtCreateSessionOptions(size_t graph_optimization_level, RETURN_NULLPTR_IF_ERROR(EnableOrtCustomOps, session_options); #endif - return session_options; + return UNREGISTER_AUTO_RELEASE(session_options); } int OrtAppendExecutionProvider(ort_session_options_handle_t session_options, const char* name) { @@ -136,11 +163,6 @@ void OrtReleaseSessionOptions(OrtSessionOptions* session_options) { } OrtSession* OrtCreateSession(void* data, size_t data_length, OrtSessionOptions* session_options) { - // OrtSessionOptions must not be nullptr. - if (session_options == nullptr) { - return nullptr; - } - #if defined(__EMSCRIPTEN_PTHREADS__) RETURN_NULLPTR_IF_ERROR(DisablePerSessionThreads, session_options); #else @@ -159,14 +181,10 @@ void OrtReleaseSession(OrtSession* session) { Ort::GetApi().ReleaseSession(session); } -size_t OrtGetInputCount(OrtSession* session) { - size_t input_count = 0; - return (CHECK_STATUS(SessionGetInputCount, session, &input_count) == ORT_OK) ? input_count : 0; -} - -size_t OrtGetOutputCount(OrtSession* session) { - size_t output_count = 0; - return (CHECK_STATUS(SessionGetOutputCount, session, &output_count) == ORT_OK) ? output_count : 0; +int OrtGetInputOutputCount(OrtSession* session, size_t* input_count, size_t* output_count) { + RETURN_ERROR_CODE_IF_ERROR(SessionGetInputCount, session, input_count); + RETURN_ERROR_CODE_IF_ERROR(SessionGetOutputCount, session, output_count); + return ORT_OK; } char* OrtGetInputName(OrtSession* session, size_t index) { @@ -210,100 +228,79 @@ OrtValue* OrtCreateTensor(int data_type, void* data, size_t data_length, size_t* RETURN_NULLPTR_IF_ERROR(CreateTensorAsOrtValue, allocator, dims_length > 0 ? shapes.data() : nullptr, dims_length, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, &value); + REGISTER_AUTO_RELEASE_HANDLE(Value, value); const char* const* strings = reinterpret_cast(data); RETURN_NULLPTR_IF_ERROR(FillStringTensor, value, strings, data_length / sizeof(const char*)); - return value; + return UNREGISTER_AUTO_RELEASE(value); } else { OrtMemoryInfo* memoryInfo = nullptr; RETURN_NULLPTR_IF_ERROR(CreateCpuMemoryInfo, OrtDeviceAllocator, OrtMemTypeDefault, &memoryInfo); + REGISTER_AUTO_RELEASE_HANDLE(MemoryInfo, memoryInfo); OrtValue* value = nullptr; int error_code = CHECK_STATUS(CreateTensorWithDataAsOrtValue, memoryInfo, data, data_length, dims_length > 0 ? shapes.data() : nullptr, dims_length, static_cast(data_type), &value); - Ort::GetApi().ReleaseMemoryInfo(memoryInfo); return (error_code == ORT_OK) ? value : nullptr; } } int OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dims, size_t* dims_length) { -#define RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \ - do { \ - int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \ - if (error_code != ORT_OK) { \ - if (info != nullptr) { \ - Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); \ - } \ - if (allocator != nullptr && p_dims != nullptr) { \ - allocator->Free(allocator, p_dims); \ - } \ - if (allocator != nullptr && p_string_data != nullptr) { \ - allocator->Free(allocator, p_string_data); \ - } \ - return error_code; \ - } \ - } while (false) - - OrtTensorTypeAndShapeInfo* info = nullptr; - OrtAllocator* allocator = nullptr; - size_t* p_dims = nullptr; - void* p_string_data = nullptr; - ONNXType tensor_type; RETURN_ERROR_CODE_IF_ERROR(GetValueType, tensor, &tensor_type); if (tensor_type != ONNX_TYPE_TENSOR) { - return ORT_FAIL; + return CheckStatus( + Ort::GetApi().CreateStatus(ORT_NOT_IMPLEMENTED, "Reading data from non-tensor typed value is not supported.")); } + OrtTensorTypeAndShapeInfo* info = nullptr; RETURN_ERROR_CODE_IF_ERROR(GetTensorTypeAndShape, tensor, &info); + REGISTER_AUTO_RELEASE_HANDLE(TensorTypeAndShapeInfo, info); size_t dims_len = 0; - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetDimensionsCount, info, &dims_len); + RETURN_ERROR_CODE_IF_ERROR(GetDimensionsCount, info, &dims_len); - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator); - p_dims = reinterpret_cast(allocator->Alloc(allocator, sizeof(size_t) * dims_len)); + OrtAllocator* allocator = nullptr; + RETURN_ERROR_CODE_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator); - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorMutableData, tensor, data); + size_t* p_dims = reinterpret_cast(allocator->Alloc(allocator, sizeof(size_t) * dims_len)); + REGISTER_AUTO_RELEASE_BUFFER(size_t, p_dims, allocator); ONNXTensorElementDataType type; - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorElementType, info, &type); - *data_type = static_cast(type); + RETURN_ERROR_CODE_IF_ERROR(GetTensorElementType, info, &type); - *dims_length = dims_len; std::vector shape(dims_len, 0); - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetDimensions, info, shape.data(), shape.size()); + RETURN_ERROR_CODE_IF_ERROR(GetDimensions, info, shape.data(), shape.size()); for (size_t i = 0; i < dims_len; i++) { p_dims[i] = static_cast(shape[i]); } - *dims = p_dims; if (type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { size_t num_elements; - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorShapeElementCount, info, &num_elements); + RETURN_ERROR_CODE_IF_ERROR(GetTensorShapeElementCount, info, &num_elements); // NOTE: ORT C-API does not expose an interface for users to get string raw data directly. There is always a copy. - // we can use the tensor raw data because it is type of "std::string *", which is very starightforward to + // we can use the tensor raw data because it is type of "std::string *", which is very straightforward to // implement and can also save memory usage. However, this approach depends on the Tensor's implementation // details. So we have to copy the string content here. size_t string_data_length; - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetStringTensorDataLength, tensor, &string_data_length); + RETURN_ERROR_CODE_IF_ERROR(GetStringTensorDataLength, tensor, &string_data_length); // The buffer contains following data: // - a sequence of pointers to (const char*), size = num_elements * sizeof(const char*). // - followed by a raw buffer to store string content, size = string_data_length + 1. - static_assert(sizeof(const char*) == sizeof(size_t), "size of a pointer and a size_t value should be the same."); - size_t string_data_offset = num_elements * sizeof(const char*); size_t buf_size = string_data_offset + string_data_length; - p_string_data = allocator->Alloc(allocator, buf_size + 1); + void* p_string_data = allocator->Alloc(allocator, buf_size + 1); void* p_string_content = reinterpret_cast(p_string_data) + string_data_offset; + REGISTER_AUTO_RELEASE_BUFFER(void, p_string_data, allocator); size_t* p_offsets = reinterpret_cast(p_string_data); - RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetStringTensorContent, tensor, p_string_content, string_data_length, p_offsets, num_elements); + RETURN_ERROR_CODE_IF_ERROR(GetStringTensorContent, tensor, p_string_content, string_data_length, p_offsets, num_elements); // replace offsets by pointers const char** p_c_strs = reinterpret_cast(p_offsets); @@ -313,10 +310,17 @@ int OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dim // put null at the last char reinterpret_cast(p_string_data)[buf_size] = '\0'; - *data = p_string_data; + + *data = UNREGISTER_AUTO_RELEASE(p_string_data); + } else { + void* p_tensor_raw_data = nullptr; + RETURN_ERROR_CODE_IF_ERROR(GetTensorMutableData, tensor, &p_tensor_raw_data); + *data = p_tensor_raw_data; } - Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); + *data_type = static_cast(type); + *dims_length = dims_len; + *dims = UNREGISTER_AUTO_RELEASE(p_dims); return ORT_OK; } @@ -330,6 +334,7 @@ OrtRunOptions* OrtCreateRunOptions(size_t log_severity_level, const char* tag) { OrtRunOptions* run_options = nullptr; RETURN_NULLPTR_IF_ERROR(CreateRunOptions, &run_options); + REGISTER_AUTO_RELEASE_HANDLE(RunOptions, run_options); // Assume that a logging level is check and properly set at JavaScript RETURN_NULLPTR_IF_ERROR(RunOptionsSetRunLogSeverityLevel, run_options, log_severity_level); @@ -346,7 +351,7 @@ OrtRunOptions* OrtCreateRunOptions(size_t log_severity_level, RETURN_NULLPTR_IF_ERROR(RunOptionsSetRunTag, run_options, tag); } - return run_options; + return UNREGISTER_AUTO_RELEASE(run_options); } int OrtAddRunConfigEntry(OrtRunOptions* run_options, diff --git a/onnxruntime/wasm/api.h b/onnxruntime/wasm/api.h index 80466ecd87..5494a9e1b4 100644 --- a/onnxruntime/wasm/api.h +++ b/onnxruntime/wasm/api.h @@ -30,9 +30,17 @@ extern "C" { * perform global initialization. should be called only once. * @param num_threads number of total threads to use. * @param logging_level default logging level. + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtInit(int num_threads, int logging_level); +/** + * get the last error. + * @param error_code [out] a pointer to accept the error code. + * @param error_message [out] a pointer to accept the error message. The message buffer is only available before any ORT API is called. + */ +void EMSCRIPTEN_KEEPALIVE OrtGetLastError(int* error_code, const char** error_message); + /** * create an instance of ORT session options. * assume that all enum type parameters, such as graph_optimization_level, execution_mode, and log_severity_level, @@ -47,7 +55,7 @@ int EMSCRIPTEN_KEEPALIVE OrtInit(int num_threads, int logging_level); * @param log_severity_level verbose, info, warning, error or fatal * @param log_verbosity_level vlog level * @param optimized_model_filepath filepath of the optimized model to dump. - * @returns a pointer to a session option handle and must be freed by calling OrtReleaseSessionOptions(). + * @returns a session option handle. Caller must release it after use by calling OrtReleaseSessionOptions(). */ ort_session_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSessionOptions(size_t graph_optimization_level, bool enable_cpu_mem_arena, @@ -63,6 +71,7 @@ ort_session_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSessionOptions(size_t /** * append an execution provider for a session. * @param name the name of the execution provider + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtAppendExecutionProvider(ort_session_options_handle_t session_options, const char* name); @@ -73,6 +82,7 @@ int EMSCRIPTEN_KEEPALIVE OrtAppendExecutionProvider(ort_session_options_handle_t * @param config_key configuration keys and value formats are defined in * include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h * @param config_value value for config_key + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtAddSessionConfigEntry(ort_session_options_handle_t session_options, const char* config_key, @@ -87,7 +97,7 @@ void EMSCRIPTEN_KEEPALIVE OrtReleaseSessionOptions(ort_session_options_handle_t * create an instance of ORT session. * @param data a pointer to a buffer that contains the ONNX or ORT format model. * @param data_length the size of the buffer in bytes. - * @returns a handle of the ORT session. + * @returns an ORT session handle. Caller must release it after use by calling OrtReleaseSession(). */ ort_session_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSession(void* data, size_t data_length, @@ -98,8 +108,16 @@ ort_session_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSession(void* data, */ void EMSCRIPTEN_KEEPALIVE OrtReleaseSession(ort_session_handle_t session); -size_t EMSCRIPTEN_KEEPALIVE OrtGetInputCount(ort_session_handle_t session); -size_t EMSCRIPTEN_KEEPALIVE OrtGetOutputCount(ort_session_handle_t session); +/** + * get model's input count and output count. + * @param session handle of the specified session + * @param input_count [out] a pointer to a size_t variable to accept input_count. + * @param output_count [out] a pointer to a size_t variable to accept output_count. + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. + */ +int EMSCRIPTEN_KEEPALIVE OrtGetInputOutputCount(ort_session_handle_t session, + size_t* input_count, + size_t* output_count); /** * get the model's input name. @@ -131,7 +149,7 @@ void EMSCRIPTEN_KEEPALIVE OrtFree(void* ptr); * @param data_length size of the buffer 'data' in bytes. * @param dims a pointer to an array of dims. the array should contain (dims_length) element(s). * @param dims_length the length of the tensor's dimension - * @returns a handle of the tensor. + * @returns a tensor handle. Caller must release it after use by calling OrtReleaseTensor(). */ ort_tensor_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateTensor(int data_type, void* data, size_t data_length, size_t* dims, size_t dims_length); @@ -144,6 +162,7 @@ ort_tensor_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateTensor(int data_type, void* da * @param dims_length [out] specify the memory to write dims length * @remarks following temporary buffers are allocated during the call. Caller must release the buffers after use by calling OrtFree(): * 'dims' (for all types of tensor), 'data' (only for string tensor) + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtGetTensorData(ort_tensor_handle_t tensor, int* data_type, void** data, size_t** dims, size_t* dims_length); @@ -158,7 +177,7 @@ void EMSCRIPTEN_KEEPALIVE OrtReleaseTensor(ort_tensor_handle_t tensor); * @param log_verbosity_level vlog level * @param terminate if true, all incomplete OrtRun calls will exit as soon as possible * @param tag tag for this run - * @returns a pointer to a run option handle and must be freed by calling OrtReleaseRunOptions(). + * @returns a run option handle. Caller must release it after use by calling OrtReleaseRunOptions(). */ ort_run_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateRunOptions(size_t log_severity_level, size_t log_verbosity_level, @@ -171,6 +190,7 @@ ort_run_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateRunOptions(size_t log_sev * @param config_key configuration keys and value formats are defined in * include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h * @param config_value value for config_key + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtAddRunConfigEntry(ort_run_options_handle_t run_options, const char* config_key, @@ -184,7 +204,7 @@ void EMSCRIPTEN_KEEPALIVE OrtReleaseRunOptions(ort_run_options_handle_t run_opti /** * inference the model. * @param session handle of the specified session - * @returns error code defined in enum OrtErrorCode + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. */ int EMSCRIPTEN_KEEPALIVE OrtRun(ort_session_handle_t session, const char** input_names,