saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-25 19:48:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Bind existing SessionOptions and RunOptions in Javascript API with WebAssembly (#7621)

Browse source 
* support session options and run options. use onnxruntime c api.

* fix lint errors

* add an error code on throwing an exception

* resolve comments. change remaining C++ APIs to C API
This commit is contained in:
Sunghoon 2021-05-13 10:50:04 -07:00 committed by GitHub
parent 333318af04
commit 1ab8a95eb6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 597 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
js/common/lib/env.ts
View file

@ -11,6 +11,11 @@ export declare namespace Env {
*/
numThreads?: number;
/**
* set a logging level. If omitted, default is 'warning'
*/
loggingLevel?: 'verbose'|'info'|'warning'|'error'|'fatal';
/**
* Set or get a number specifying the timeout for initialization of WebAssembly backend, in milliseconds. A zero
* value indicates no timeout is set. (default is 0)

11
js/web/lib/backend-wasm.ts
View file

@ -21,6 +21,11 @@ export const initializeFlags = (): void => {
env.wasm.numThreads = Math.ceil((navigator.hardwareConcurrency || 1) / 2);
}
env.wasm.numThreads = Math.min(4, env.wasm.numThreads);
if (typeof env.wasm.loggingLevel !== 'string' ||
['verbose', 'info', 'warning', 'error', 'fatal'].indexOf(env.wasm.loggingLevel) === -1) {
env.wasm.loggingLevel = 'warning';
}
};
class OnnxruntimeWebAssemblyBackend implements Backend {
@ -33,7 +38,7 @@ class OnnxruntimeWebAssemblyBackend implements Backend {
}
createSessionHandler(path: string, options?: InferenceSession.SessionOptions): Promise<SessionHandler>;
createSessionHandler(buffer: Uint8Array, options?: InferenceSession.SessionOptions): Promise<SessionHandler>;
async createSessionHandler(pathOrBuffer: string|Uint8Array, _options?: InferenceSession.SessionOptions):
async createSessionHandler(pathOrBuffer: string|Uint8Array, options?: InferenceSession.SessionOptions):
Promise<SessionHandler> {
let buffer: Uint8Array;
if (typeof pathOrBuffer === 'string') {
@ -43,9 +48,9 @@ class OnnxruntimeWebAssemblyBackend implements Backend {
} else {
buffer = pathOrBuffer;
}
const handler = new OnnxruntimeWebAssemblySessionHandler();
// TODO: support SessionOptions
handler.loadModel(buffer);
handler.loadModel(buffer, options);
return Promise.resolve(handler);
}
}

24
js/web/lib/wasm/binding/ort-wasm.d.ts vendored
View file

@ -13,9 +13,9 @@ export interface OrtWasmModule extends EmscriptenModule {
//#endregion
//#region ORT APIs
_OrtInit(numThreads: number, loggingLevel: number): void;
_OrtInit(numThreads: number, loggingLevel: number): number;
_OrtCreateSession(dataOffset: number, dataLength: number): number;
_OrtCreateSession(dataOffset: number, dataLength: number, sessionOptionsHandle: number): number;
_OrtReleaseSession(sessionHandle: number): void;
_OrtGetInputCount(sessionHandle: number): number;
_OrtGetOutputCount(sessionHandle: number): number;
@ -27,11 +27,27 @@ export interface OrtWasmModule extends EmscriptenModule {
_OrtCreateTensor(dataType: number, dataOffset: number, dataLength: number, dimsOffset: number, dimsLength: number):
number;
_OrtGetTensorData(tensorHandle: number, dataType: number, dataOffset: number, dimsOffset: number, dimsLength: number):
void;
number;
_OrtReleaseTensor(tensorHandle: number): void;
_OrtRun(
sessionHandle: number, inputNamesOffset: number, inputsOffset: number, inputCount: number,
outputNamesOffset: number, outputCount: number, outputsOffset: number): number;
outputNamesOffset: number, outputCount: number, outputsOffset: number, runOptionsHandle: number): number;
_OrtCreateSessionOptions(): number;
_OrtReleaseSessionOptions(sessionOptionsHandle: number): void;
_OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle: number, level: number): number;
_OrtEnableCpuMemArena(sessionOptionsHandle: number): number;
_OrtDisableCpuMemArena(sessionOptionsHandle: number): number;
_OrtEnableMemPattern(sessionOptionsHandle: number): number;
_OrtDisableMemPattern(sessionOptionsHandle: number): number;
_OrtSetSessionExecutionMode(sessionOptionsHandle: number, mode: number): number;
_OrtSetSessionLogId(sessionOptionsHandle: number, logid: number): number;
_OrtSetSessionLogSeverityLevel(sessionOptionsHandle: number, level: number): number;
_OrtCreateRunOptions(): number;
_OrtReleaseRunOptions(runOptionsHandle: number): void;
_OrtRunOptionsSetRunLogSeverityLevel(runOptionsHandle: number, level: number): number;
_OrtRunOptionsSetRunTag(runOptionsHandle: number, tag: number): number;
//#endregion
//#region config

373
js/web/lib/wasm/session-handler.ts
View file

@ -102,6 +102,150 @@ const numericTensorTypeToTypedArray = (type: Tensor.Type): Float32ArrayConstruct
}
};
const getLoggingLevel = (loggingLevel: 'verbose'|'info'|'warning'|'error'|'fatal'): number => {
switch (loggingLevel) {
case 'verbose':
return 0;
case 'info':
return 1;
case 'warning':
return 2;
case 'error':
return 3;
case 'fatal':
return 4;
default:
throw new Error(`unsupported logging level: ${loggingLevel}`);
}
};
const setSessionOptions = (options?: InferenceSession.SessionOptions): [number, number[]] => {
const wasm = getInstance();
const sessionOptionsHandle = wasm._OrtCreateSessionOptions();
const allocs: number[] = [];
if (sessionOptionsHandle === 0) {
throw new Error('Can\'t create session options');
}
if (options === undefined) {
return [sessionOptionsHandle, allocs];
}
let errorCode = 0;
if (options.graphOptimizationLevel !== undefined) {
switch (options.graphOptimizationLevel) {
case 'disabled':
errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 0);
break;
case 'basic':
errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 1);
break;
case 'extended':
errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 2);
break;
case 'all':
errorCode = wasm._OrtSetSessionGraphOptimizationLevel(sessionOptionsHandle, 99);
break;
default:
throw new Error(`unsupported graph optimization level: ${options.graphOptimizationLevel}`);
}
if (errorCode !== 0) {
throw new Error(`Can't set a graph optimization level as a session option. error code = ${errorCode}`);
}
}
if (options.enableCpuMemArena !== undefined) {
if (options.enableCpuMemArena) {
errorCode = wasm._OrtEnableCpuMemArena(sessionOptionsHandle);
} else {
errorCode = wasm._OrtDisableCpuMemArena(sessionOptionsHandle);
}
if (errorCode !== 0) {
throw new Error(`Can't set a CPU memory arena as a session option. error code = ${errorCode}`);
}
}
if (options.enableMemPattern !== undefined) {
if (options.enableMemPattern) {
errorCode = wasm._OrtEnableMemPattern(sessionOptionsHandle);
} else {
errorCode = wasm._OrtDisableMemPattern(sessionOptionsHandle);
}
if (errorCode !== 0) {
throw new Error(`Can't set a memory pattern as a session option. error code = ${errorCode}`);
}
}
if (options.executionMode !== undefined) {
switch (options.executionMode) {
case 'sequential':
errorCode = wasm._OrtSetSessionExecutionMode(sessionOptionsHandle, 0);
break;
case 'parallel':
errorCode = wasm._OrtSetSessionExecutionMode(sessionOptionsHandle, 1);
break;
default:
throw new Error(`unsupported execution mode: ${options.executionMode}`);
}
if (errorCode !== 0) {
throw new Error(`Can't set an execution mode as a session option. error code = ${errorCode}`);
}
}
if (options.logId !== undefined) {
const logIdDataLength = wasm.lengthBytesUTF8(options.logId) + 1;
const logIdDataOffset = wasm._malloc(logIdDataLength);
wasm.stringToUTF8(options.logId, logIdDataOffset, logIdDataLength);
errorCode = wasm._OrtSetSessionLogId(sessionOptionsHandle, logIdDataOffset);
allocs.push(logIdDataOffset);
if (errorCode !== 0) {
throw new Error(`Can't set a log id as a session option. error code = ${errorCode}`);
}
}
if (options.logSeverityLevel !== undefined) {
errorCode = wasm._OrtSetSessionLogSeverityLevel(sessionOptionsHandle, options.logSeverityLevel);
if (errorCode !== 0) {
throw new Error(`Can't set a log severity level as a session option. error code = ${errorCode}`);
}
}
return [sessionOptionsHandle, allocs];
};
const setRunOptions = (options: InferenceSession.RunOptions): [number, number[]] => {
const wasm = getInstance();
const runOptionsHandle = wasm._OrtCreateRunOptions();
if (runOptionsHandle === 0) {
throw new Error('Can\'t create run options');
}
const allocs: number[] = [];
let errorCode = 0;
if (options.logSeverityLevel !== undefined) {
errorCode = wasm._OrtRunOptionsSetRunLogSeverityLevel(runOptionsHandle, options.logSeverityLevel);
if (errorCode !== 0) {
throw new Error(`Can't set a log severity level as a run option. error code = ${errorCode}`);
}
}
if (options.tag !== undefined) {
const tagDataLength = wasm.lengthBytesUTF8(options.tag) + 1;
const tagDataOffset = wasm._malloc(tagDataLength);
wasm.stringToUTF8(options.tag, tagDataOffset, tagDataLength);
errorCode = wasm._OrtRunOptionsSetRunTag(runOptionsHandle, tagDataOffset);
allocs.push(tagDataOffset);
if (errorCode !== 0) {
throw new Error(`Can't set a tag as a run option. error code = ${errorCode}`);
}
}
return [runOptionsHandle, allocs];
};
export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
private sessionHandle: number;
@ -110,19 +254,32 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
outputNames: string[];
private outputNamesUTF8Encoded: number[];
loadModel(model: Uint8Array): void {
loadModel(model: Uint8Array, options?: InferenceSession.SessionOptions): void {
const wasm = getInstance();
if (!ortInit) {
wasm._OrtInit(env.wasm.numThreads!, 2 /* LoggingLevel::Warning */);
const errorCode = wasm._OrtInit(env.wasm.numThreads!, getLoggingLevel(env.wasm.loggingLevel!));
if (errorCode !== 0) {
throw new Error(`Can't initialize onnxruntime. error code = ${errorCode}`);
}
ortInit = true;
}
const modelDataOffset = wasm._malloc(model.byteLength);
let sessionOptionsHandle = 0;
let allocs: number[] = [];
try {
[sessionOptionsHandle, allocs] = setSessionOptions(options);
wasm.HEAPU8.set(model, modelDataOffset);
this.sessionHandle = wasm._OrtCreateSession(modelDataOffset, model.byteLength);
this.sessionHandle = wasm._OrtCreateSession(modelDataOffset, model.byteLength, sessionOptionsHandle);
if (this.sessionHandle === 0) {
throw new Error('Can\'t create a session');
}
} finally {
wasm._free(modelDataOffset);
wasm._OrtReleaseSessionOptions(sessionOptionsHandle);
allocs.forEach(wasm._free);
}
const inputCount = wasm._OrtGetInputCount(this.sessionHandle);
@ -134,11 +291,17 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
this.outputNamesUTF8Encoded = [];
for (let i = 0; i < inputCount; i++) {
const name = wasm._OrtGetInputName(this.sessionHandle, i);
if (name === 0) {
throw new Error('Can\'t get an input name');
}
this.inputNamesUTF8Encoded.push(name);
this.inputNames.push(wasm.UTF8ToString(name));
}
for (let i = 0; i < outputCount; i++) {
const name = wasm._OrtGetOutputName(this.sessionHandle, i);
if (name === 0) {
throw new Error('Can\'t get an output name');
}
this.outputNamesUTF8Encoded.push(name);
this.outputNames.push(wasm.UTF8ToString(name));
}
@ -147,11 +310,11 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
async dispose(): Promise<void> {
const wasm = getInstance();
if (this.inputNamesUTF8Encoded) {
this.inputNamesUTF8Encoded.forEach(str => wasm._OrtFree(str));
this.inputNamesUTF8Encoded.forEach(wasm._OrtFree);
this.inputNamesUTF8Encoded = [];
}
if (this.outputNamesUTF8Encoded) {
this.outputNamesUTF8Encoded.forEach(str => wasm._OrtFree(str));
this.outputNamesUTF8Encoded.forEach(wasm._OrtFree);
this.outputNamesUTF8Encoded = [];
}
if (this.sessionHandle) {
@ -160,9 +323,8 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
}
}
async run(
feeds: SessionHandler.FeedsType, fetches: SessionHandler.FetchesType,
_options: InferenceSession.RunOptions): Promise<SessionHandler.ReturnType> {
async run(feeds: SessionHandler.FeedsType, fetches: SessionHandler.FetchesType, options: InferenceSession.RunOptions):
Promise<SessionHandler.ReturnType> {
const wasm = getInstance();
const inputArray: Tensor[] = [];
@ -196,112 +358,129 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
const inputCount = inputIndices.length;
const outputCount = outputIndices.length;
let runOptionsHandle = 0;
let allocs: number[] = [];
const inputValues: number[] = [];
const inputDataOffsets: number[] = [];
// create input tensors
for (let i = 0; i < inputCount; i++) {
const data = inputArray[i].data;
if (Array.isArray(data)) {
// string tensor
throw new TypeError('string tensor is not supported');
} else {
const dataOffset = wasm._malloc(data.byteLength);
inputDataOffsets.push(dataOffset);
wasm.HEAPU8.set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), dataOffset);
const dims = inputArray[i].dims;
const stack = wasm.stackSave();
const dimsOffset = wasm.stackAlloc(4 * dims.length);
try {
let dimIndex = dimsOffset / 4;
dims.forEach(d => wasm.HEAP32[dimIndex++] = d);
const tensor = wasm._OrtCreateTensor(
tensorDataTypeStringToEnum(inputArray[i].type), dataOffset, data.byteLength, dimsOffset, dims.length);
inputValues.push(tensor);
} finally {
wasm.stackRestore(stack);
}
}
}
const beforeRunStack = wasm.stackSave();
const inputValuesOffset = wasm.stackAlloc(inputCount * 4);
const inputNamesOffset = wasm.stackAlloc(inputCount * 4);
const outputValuesOffset = wasm.stackAlloc(outputCount * 4);
const outputNamesOffset = wasm.stackAlloc(outputCount * 4);
try {
let inputValuesIndex = inputValuesOffset / 4;
let inputNamesIndex = inputNamesOffset / 4;
let outputValuesIndex = outputValuesOffset / 4;
let outputNamesIndex = outputNamesOffset / 4;
[runOptionsHandle, allocs] = setRunOptions(options);
// create input tensors
for (let i = 0; i < inputCount; i++) {
wasm.HEAPU32[inputValuesIndex++] = inputValues[i];
wasm.HEAPU32[inputNamesIndex++] = this.inputNamesUTF8Encoded[inputIndices[i]];
}
for (let i = 0; i < outputCount; i++) {
wasm.HEAPU32[outputValuesIndex++] = 0;
wasm.HEAPU32[outputNamesIndex++] = this.outputNamesUTF8Encoded[outputIndices[i]];
}
const data = inputArray[i].data;
if (Array.isArray(data)) {
// string tensor
throw new TypeError('string tensor is not supported');
} else {
const dataOffset = wasm._malloc(data.byteLength);
inputDataOffsets.push(dataOffset);
wasm.HEAPU8.set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), dataOffset);
// support RunOptions
const errorCode = wasm._OrtRun(
this.sessionHandle, inputNamesOffset, inputValuesOffset, inputCount, outputNamesOffset, outputCount,
outputValuesOffset);
const dims = inputArray[i].dims;
const output: {[name: string]: Tensor} = {};
if (errorCode === 0) {
for (let i = 0; i < outputCount; i++) {
const tensor = wasm.HEAPU32[outputValuesOffset / 4 + i];
const beforeGetTensorDataStack = wasm.stackSave();
// stack allocate 4 pointer value
const tensorDataOffset = wasm.stackAlloc(4 * 4);
const stack = wasm.stackSave();
const dimsOffset = wasm.stackAlloc(4 * dims.length);
try {
wasm._OrtGetTensorData(
tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12);
let tensorDataIndex = tensorDataOffset / 4;
const dataType = wasm.HEAPU32[tensorDataIndex++];
const dataOffset: number = wasm.HEAPU32[tensorDataIndex++];
const dimsOffset = wasm.HEAPU32[tensorDataIndex++];
const dimsLength = wasm.HEAPU32[tensorDataIndex++];
const dims = [];
for (let i = 0; i < dimsLength; i++) {
dims.push(wasm.HEAPU32[dimsOffset / 4 + i]);
}
wasm._OrtFree(dimsOffset);
const type = tensorDataTypeEnumToString(dataType);
if (type === 'string') {
// string tensor
throw new TypeError('string tensor is not supported');
} else {
const typedArray = numericTensorTypeToTypedArray(type);
const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b);
const t = new Tensor(type, new typedArray(size), dims) as TypedTensor<Exclude<Tensor.Type, 'string'>>;
new Uint8Array(t.data.buffer, t.data.byteOffset, t.data.byteLength)
.set(wasm.HEAPU8.subarray(dataOffset, dataOffset + t.data.byteLength));
output[this.outputNames[outputIndices[i]]] = t;
let dimIndex = dimsOffset / 4;
dims.forEach(d => wasm.HEAP32[dimIndex++] = d);
const tensor = wasm._OrtCreateTensor(
tensorDataTypeStringToEnum(inputArray[i].type), dataOffset, data.byteLength, dimsOffset, dims.length);
if (tensor === 0) {
throw new Error('Can\'t create a tensor');
}
inputValues.push(tensor);
} finally {
wasm.stackRestore(beforeGetTensorDataStack);
wasm.stackRestore(stack);
}
wasm._OrtReleaseTensor(tensor);
}
}
inputValues.forEach(t => wasm._OrtReleaseTensor(t));
inputDataOffsets.forEach(i => wasm._free(i));
const beforeRunStack = wasm.stackSave();
const inputValuesOffset = wasm.stackAlloc(inputCount * 4);
const inputNamesOffset = wasm.stackAlloc(inputCount * 4);
const outputValuesOffset = wasm.stackAlloc(outputCount * 4);
const outputNamesOffset = wasm.stackAlloc(outputCount * 4);
if (errorCode === 0) {
return output;
} else {
throw new Error(`failed to call OrtRun(). error code = ${errorCode}.`);
try {
let inputValuesIndex = inputValuesOffset / 4;
let inputNamesIndex = inputNamesOffset / 4;
let outputValuesIndex = outputValuesOffset / 4;
let outputNamesIndex = outputNamesOffset / 4;
for (let i = 0; i < inputCount; i++) {
wasm.HEAPU32[inputValuesIndex++] = inputValues[i];
wasm.HEAPU32[inputNamesIndex++] = this.inputNamesUTF8Encoded[inputIndices[i]];
}
for (let i = 0; i < outputCount; i++) {
wasm.HEAPU32[outputValuesIndex++] = 0;
wasm.HEAPU32[outputNamesIndex++] = this.outputNamesUTF8Encoded[outputIndices[i]];
}
// support RunOptions
let errorCode = wasm._OrtRun(
this.sessionHandle, inputNamesOffset, inputValuesOffset, inputCount, outputNamesOffset, outputCount,
outputValuesOffset, runOptionsHandle);
const output: {[name: string]: Tensor} = {};
if (errorCode === 0) {
for (let i = 0; i < outputCount; i++) {
const tensor = wasm.HEAPU32[outputValuesOffset / 4 + i];
const beforeGetTensorDataStack = wasm.stackSave();
// stack allocate 4 pointer value
const tensorDataOffset = wasm.stackAlloc(4 * 4);
try {
errorCode = wasm._OrtGetTensorData(
tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12);
if (errorCode !== 0) {
throw new Error(`Can't get a tensor data. error code = ${errorCode}`);
}
let tensorDataIndex = tensorDataOffset / 4;
const dataType = wasm.HEAPU32[tensorDataIndex++];
const dataOffset: number = wasm.HEAPU32[tensorDataIndex++];
const dimsOffset = wasm.HEAPU32[tensorDataIndex++];
const dimsLength = wasm.HEAPU32[tensorDataIndex++];
const dims = [];
for (let i = 0; i < dimsLength; i++) {
dims.push(wasm.HEAPU32[dimsOffset / 4 + i]);
}
wasm._OrtFree(dimsOffset);
const type = tensorDataTypeEnumToString(dataType);
if (type === 'string') {
// string tensor
throw new TypeError('string tensor is not supported');
} else {
const typedArray = numericTensorTypeToTypedArray(type);
const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b);
const t = new Tensor(type, new typedArray(size), dims) as TypedTensor<Exclude<Tensor.Type, 'string'>>;
new Uint8Array(t.data.buffer, t.data.byteOffset, t.data.byteLength)
.set(wasm.HEAPU8.subarray(dataOffset, dataOffset + t.data.byteLength));
output[this.outputNames[outputIndices[i]]] = t;
}
} finally {
wasm.stackRestore(beforeGetTensorDataStack);
wasm._OrtReleaseTensor(tensor);
}
}
}
if (errorCode === 0) {
return output;
} else {
throw new Error(`failed to call OrtRun(). error code = ${errorCode}.`);
}
} finally {
wasm.stackRestore(beforeRunStack);
}
} finally {
wasm.stackRestore(beforeRunStack);
inputValues.forEach(wasm._OrtReleaseTensor);
inputDataOffsets.forEach(wasm._free);
wasm._OrtReleaseRunOptions(runOptionsHandle);
allocs.forEach(wasm._free);
}
}

3
js/web/script/test-runner-cli.ts
View file

@ -568,6 +568,9 @@ function saveConfig(config: Test.Config) {
if (config.options.wasmFlags && config.options.wasmFlags.numThreads !== undefined) {
setOptions += `ort.env.wasm.numThreads = ${JSON.stringify(config.options.wasmFlags.numThreads)};`;
}
if (config.options.wasmFlags && config.options.wasmFlags.loggingLevel !== undefined) {
setOptions += `ort.env.wasm.loggingLevel = ${JSON.stringify(config.options.wasmFlags.loggingLevel)};`;
}
if (config.options.wasmFlags && config.options.wasmFlags.initTimeout !== undefined) {
setOptions += `ort.env.wasm.initTimeout = ${JSON.stringify(config.options.wasmFlags.initTimeout)};`;
}

260
onnxruntime/wasm/api.cc
View file

@ -9,60 +9,156 @@
#include <vector>
namespace {
Ort::Env* g_env;
OrtEnv* g_env;
} // namespace
void OrtInit(int numThreads, int logging_level) {
#if defined(__EMSCRIPTEN_PTHREADS__)
OrtThreadingOptions* tp_options;
Ort::ThrowOnError(Ort::GetApi().CreateThreadingOptions(&tp_options));
Ort::ThrowOnError(Ort::GetApi().SetGlobalIntraOpNumThreads(tp_options, numThreads));
Ort::ThrowOnError(Ort::GetApi().SetGlobalInterOpNumThreads(tp_options, 1));
g_env = new Ort::Env{tp_options, static_cast<OrtLoggingLevel>(logging_level), "Default"};
#endif
g_env = new Ort::Env{static_cast<OrtLoggingLevel>(logging_level), "Default"};
OrtErrorCode CheckStatus(OrtStatusPtr status) {
OrtErrorCode error_code = ORT_OK;
if (status) {
std::string error_message = Ort::GetApi().GetErrorMessage(status);
error_code = Ort::GetApi().GetErrorCode(status);
std::cerr << Ort::Exception(std::move(error_message), error_code).what() << std::endl;
Ort::GetApi().ReleaseStatus(status);
}
return error_code;
}
Ort::Session* OrtCreateSession(void* data, size_t data_length) {
Ort::SessionOptions session_options;
session_options.SetLogId("onnxruntime");
#define CHECK_STATUS(ORT_API_NAME, ...) \
CheckStatus(Ort::GetApi().ORT_API_NAME(__VA_ARGS__))
#define RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \
do { \
int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \
if (error_code != ORT_OK) { \
return error_code; \
} \
} while (false)
// TODO: This macro can be removed when we changed all APIs to return a status code.
#define RETURN_NULLPTR_IF_ERROR(ORT_API_NAME, ...) \
do { \
if (CHECK_STATUS(ORT_API_NAME, __VA_ARGS__) != ORT_OK) { \
return nullptr; \
} \
} while (false)
int OrtInit(int num_threads, int logging_level) {
// Assume that a logging level is check and properly set at JavaScript
#if defined(__EMSCRIPTEN_PTHREADS__)
OrtThreadingOptions* tp_options = nullptr;
RETURN_ERROR_CODE_IF_ERROR(CreateThreadingOptions, &tp_options);
RETURN_ERROR_CODE_IF_ERROR(SetGlobalIntraOpNumThreads, tp_options, num_threads);
RETURN_ERROR_CODE_IF_ERROR(SetGlobalInterOpNumThreads, tp_options, 1);
return CHECK_STATUS(CreateEnvWithGlobalThreadPools,
static_cast<OrtLoggingLevel>(logging_level),
"Default",
tp_options,
&g_env);
#else
return CHECK_STATUS(CreateEnv, static_cast<OrtLoggingLevel>(logging_level), "Default", &g_env);
#endif
}
OrtSessionOptions* OrtCreateSessionOptions() {
OrtSessionOptions* session_options = nullptr;
return (CHECK_STATUS(CreateSessionOptions, &session_options) == ORT_OK) ? session_options : nullptr;
}
void OrtReleaseSessionOptions(OrtSessionOptions* session_options) {
Ort::GetApi().ReleaseSessionOptions(session_options);
}
int OrtSetSessionGraphOptimizationLevel(OrtSessionOptions* session_options, size_t level) {
// Assume that a graph optimization level is check and properly set at JavaScript
return CHECK_STATUS(SetSessionGraphOptimizationLevel, session_options, static_cast<GraphOptimizationLevel>(level));
}
int OrtEnableCpuMemArena(OrtSessionOptions* session_options) {
return CHECK_STATUS(EnableCpuMemArena, session_options);
}
int OrtDisableCpuMemArena(OrtSessionOptions* session_options) {
return CHECK_STATUS(DisableCpuMemArena, session_options);
}
int OrtEnableMemPattern(OrtSessionOptions* session_options) {
return CHECK_STATUS(EnableMemPattern, session_options);
}
int OrtDisableMemPattern(OrtSessionOptions* session_options) {
return CHECK_STATUS(DisableMemPattern, session_options);
}
int OrtSetSessionExecutionMode(OrtSessionOptions* session_options, size_t mode) {
// Assume that an execution mode is check and properly set at JavaScript
return CHECK_STATUS(SetSessionExecutionMode, session_options, static_cast<ExecutionMode>(mode));
}
int OrtSetSessionLogId(OrtSessionOptions* session_options, const char* logid) {
return CHECK_STATUS(SetSessionLogId, session_options, logid);
}
int OrtSetSessionLogSeverityLevel(OrtSessionOptions* session_options, size_t level) {
return CHECK_STATUS(SetSessionLogSeverityLevel, session_options, level);
}
OrtSession* OrtCreateSession(void* data, size_t data_length, OrtSessionOptions* session_options) {
// OrtSessionOptions must not be nullptr.
if (session_options == nullptr) {
return nullptr;
}
#if defined(__EMSCRIPTEN_PTHREADS__)
session_options.DisablePerSessionThreads();
RETURN_NULLPTR_IF_ERROR(DisablePerSessionThreads, session_options);
#else
// must disable thread pool when WebAssembly multi-threads support is disabled.
session_options.SetIntraOpNumThreads(1);
RETURN_NULLPTR_IF_ERROR(SetIntraOpNumThreads, session_options, 1);
RETURN_NULLPTR_IF_ERROR(SetSessionExecutionMode, session_options, ORT_SEQUENTIAL);
#endif
return new Ort::Session(*g_env, data, data_length, session_options);
OrtSession* session = nullptr;
return (CHECK_STATUS(CreateSessionFromArray, g_env, data, data_length, session_options, &session) == ORT_OK)
? session : nullptr;
}
void OrtReleaseSession(Ort::Session* session) {
delete session;
void OrtReleaseSession(OrtSession* session) {
Ort::GetApi().ReleaseSession(session);
}
size_t OrtGetInputCount(Ort::Session* session) {
return session->GetInputCount();
size_t OrtGetInputCount(OrtSession* session) {
size_t input_count = 0;
return (CHECK_STATUS(SessionGetInputCount, session, &input_count) == ORT_OK) ? input_count : 0;
}
size_t OrtGetOutputCount(Ort::Session* session) {
return session->GetOutputCount();
size_t OrtGetOutputCount(OrtSession* session) {
size_t output_count = 0;
return (CHECK_STATUS(SessionGetOutputCount, session, &output_count) == ORT_OK) ? output_count : 0;
}
char* OrtGetInputName(Ort::Session* session, size_t index) {
Ort::AllocatorWithDefaultOptions allocator;
return session->GetInputName(index, allocator);
char* OrtGetInputName(OrtSession* session, size_t index) {
OrtAllocator* allocator = nullptr;
RETURN_NULLPTR_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator);
char* input_name = nullptr;
return (CHECK_STATUS(SessionGetInputName, session, index, allocator, &input_name) == ORT_OK)
? input_name : nullptr;
}
char* OrtGetOutputName(Ort::Session* session, size_t index) {
Ort::AllocatorWithDefaultOptions allocator;
return session->GetOutputName(index, allocator);
char* OrtGetOutputName(OrtSession* session, size_t index) {
OrtAllocator* allocator = nullptr;
RETURN_NULLPTR_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator);
char* output_name = nullptr;
return (CHECK_STATUS(SessionGetOutputName, session, index, allocator, &output_name) == ORT_OK)
? output_name : nullptr;
}
void OrtFree(void* ptr) {
Ort::AllocatorWithDefaultOptions allocator;
allocator.Free(ptr);
OrtAllocator* allocator = nullptr;
if (CHECK_STATUS(GetAllocatorWithDefaultOptions, &allocator) == ORT_OK) {
allocator->Free(allocator, ptr);
}
}
OrtValue* OrtCreateTensor(int data_type, void* data, size_t data_length, size_t* dims, size_t dims_length) {
@ -71,47 +167,87 @@ OrtValue* OrtCreateTensor(int data_type, void* data, size_t data_length, size_t*
shapes[i] = dims[i];
}
return Ort::Value::CreateTensor(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault),
data,
data_length,
dims_length > 0 ? shapes.data() : nullptr,
dims_length,
static_cast<ONNXTensorElementDataType>(data_type))
.release();
OrtMemoryInfo* memoryInfo = nullptr;
RETURN_NULLPTR_IF_ERROR(CreateCpuMemoryInfo, OrtDeviceAllocator, OrtMemTypeDefault, &memoryInfo);
OrtValue* value = nullptr;
int error_code = CHECK_STATUS(CreateTensorWithDataAsOrtValue, memoryInfo, data, data_length,
dims_length > 0 ? shapes.data() : nullptr, dims_length,
static_cast<ONNXTensorElementDataType>(data_type), &value);
Ort::GetApi().ReleaseMemoryInfo(memoryInfo);
return (error_code == ORT_OK) ? value : nullptr;
}
void OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dims, size_t* dims_length) {
Ort::Value v{tensor};
auto info = v.GetTensorTypeAndShapeInfo();
size_t dims_len = info.GetDimensionsCount();
Ort::AllocatorWithDefaultOptions allocator;
size_t* p_dims = reinterpret_cast<size_t*>(allocator.Alloc(sizeof(size_t) * dims_len));
*data = v.GetTensorMutableData<void>();
*data_type = info.GetElementType();
int OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dims, size_t* dims_length) {
#define RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \
do { \
int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \
if (error_code != ORT_OK) { \
if (info != nullptr) { \
Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); \
} \
if (allocator != nullptr && p_dims != nullptr) { \
allocator->Free(allocator, p_dims); \
} \
return error_code; \
} \
} while (false)
OrtTensorTypeAndShapeInfo* info = nullptr;
OrtAllocator* allocator = nullptr;
size_t* p_dims = nullptr;
RETURN_ERROR_CODE_IF_ERROR(GetTensorTypeAndShape, tensor, &info);
size_t dims_len = 0;
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetDimensionsCount, info, &dims_len);
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator);
p_dims = reinterpret_cast<size_t*>(allocator->Alloc(allocator, sizeof(size_t) * dims_len));
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorMutableData, tensor, data);
ONNXTensorElementDataType type;
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorElementType, info, &type);
*data_type = static_cast<int>(type);
*dims_length = dims_len;
auto shape = info.GetShape();
std::vector<int64_t> shape(dims_len, 0);
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetDimensions, info, shape.data(), shape.size());
for (size_t i = 0; i < dims_len; i++) {
p_dims[i] = static_cast<size_t>(shape[i]);
}
*dims = p_dims;
v.release();
Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info);
return ORT_OK;
}
void OrtReleaseTensor(OrtValue* tensor) {
Ort::OrtRelease(tensor);
Ort::GetApi().ReleaseValue(tensor);
}
int OrtRun(Ort::Session* session,
const char** input_names, const ort_tensor_handle_t* inputs, size_t input_count,
const char** output_names, size_t output_count, ort_tensor_handle_t* outputs) {
OrtStatusPtr status = Ort::GetApi().Run(*session, Ort::RunOptions{nullptr}, input_names, inputs, input_count, output_names, output_count, outputs);
OrtErrorCode error_code = ORT_OK;
if (status) {
std::string error_message = Ort::GetApi().GetErrorMessage(status);
error_code = Ort::GetApi().GetErrorCode(status);
std::cerr << Ort::Exception(std::move(error_message), error_code).what()
<< std::endl;
Ort::GetApi().ReleaseStatus(status);
}
return error_code;
OrtRunOptions* OrtCreateRunOptions() {
OrtRunOptions* run_options = nullptr;
return (CHECK_STATUS(CreateRunOptions, &run_options) == ORT_OK) ? run_options : nullptr;
}
void OrtReleaseRunOptions(OrtRunOptions* run_options) {
Ort::GetApi().ReleaseRunOptions(run_options);
}
int OrtRunOptionsSetRunLogSeverityLevel(OrtRunOptions* run_options, size_t level) {
return CHECK_STATUS(RunOptionsSetRunLogSeverityLevel, run_options, level);
}
int OrtRunOptionsSetRunTag(OrtRunOptions* run_options, const char* tag) {
return CHECK_STATUS(RunOptionsSetRunTag, run_options, tag);
}
int OrtRun(OrtSession* session,
const char** input_names, const ort_tensor_handle_t* inputs, size_t input_count,
const char** output_names, size_t output_count, ort_tensor_handle_t* outputs,
OrtRunOptions* run_options) {
return CHECK_STATUS(Run, session, run_options, input_names, inputs, input_count, output_names, output_count, outputs);
}

95
onnxruntime/wasm/api.h
View file

@ -12,10 +12,14 @@
#include <stddef.h>
namespace Ort {
struct Session;
}
using ort_session_handle_t = Ort::Session*;
struct OrtSession;
using ort_session_handle_t = OrtSession*;
struct OrtSessionOptions;
using ort_session_options_handle_t = OrtSessionOptions*;
struct OrtRunOptions;
using ort_run_options_handle_t = OrtRunOptions*;
struct OrtValue;
using ort_tensor_handle_t = OrtValue*;
@ -27,7 +31,58 @@ extern "C" {
* @param numThreads number of total threads to use.
* @param logging_level default logging level.
*/
void EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level);
int EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level);
/**
* create an instance of ORT session options.
* @returns a pointer to a session option handle and must be freed by calling OrtReleaseSessionOptions().
*/
ort_session_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSessionOptions();
/**
* release the specified ORT session options.
*/
void EMSCRIPTEN_KEEPALIVE OrtReleaseSessionOptions(ort_session_options_handle_t session_options);
/**
* set an optimization level for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtSetSessionGraphOptimizationLevel(ort_session_options_handle_t session_options, size_t level);
/**
* enable CPU memory arena for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtEnableCpuMemArena(ort_session_options_handle_t session_options);
/**
* disable CPU memory arena for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtDisableCpuMemArena(ort_session_options_handle_t session_options);
/**
* enable memory pattern for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtEnableMemPattern(ort_session_options_handle_t session_options);
/**
* disable memory pattern for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtDisableMemPattern(ort_session_options_handle_t session_options);
/**
* set an execution mode for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtSetSessionExecutionMode(ort_session_options_handle_t session_options, size_t mode);
/**
* set a log ID for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtSetSessionLogId(ort_session_options_handle_t session_options, const char* logid);
/**
* set a log severity level for session.
*/
int EMSCRIPTEN_KEEPALIVE OrtSetSessionLogSeverityLevel(ort_session_options_handle_t session_options, size_t level);
/**
* create an instance of ORT session.
@ -35,7 +90,9 @@ void EMSCRIPTEN_KEEPALIVE OrtInit(int numThreads, int logging_level);
* @param data_length the size of the buffer in bytes.
* @returns a handle of the ORT session.
*/
ort_session_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSession(void* data, size_t data_length);
ort_session_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateSession(void* data,
size_t data_length,
ort_session_options_handle_t session_options);
/**
* release the specified ORT session.
@ -88,13 +145,34 @@ ort_tensor_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateTensor(int data_type, void* da
* @param dims_length [out] specify the memory to write dims length
* @remarks a temporary buffer 'dims' is allocated during the call. Caller must release the buffer after use by calling OrtFree().
*/
void EMSCRIPTEN_KEEPALIVE OrtGetTensorData(ort_tensor_handle_t tensor, int* data_type, void** data, size_t** dims, size_t* dims_length);
int EMSCRIPTEN_KEEPALIVE OrtGetTensorData(ort_tensor_handle_t tensor, int* data_type, void** data, size_t** dims, size_t* dims_length);
/**
* release the specified tensor.
*/
void EMSCRIPTEN_KEEPALIVE OrtReleaseTensor(ort_tensor_handle_t tensor);
/**
* create an instance of ORT run options.
* @returns a pointer to a run option handle and must be freed by calling OrtReleaseRunOptions().
*/
ort_run_options_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateRunOptions();
/**
* release the specified ORT run options.
*/
void EMSCRIPTEN_KEEPALIVE OrtReleaseRunOptions(ort_run_options_handle_t run_options);
/**
* set log severity level for run.
*/
int EMSCRIPTEN_KEEPALIVE OrtRunOptionsSetRunLogSeverityLevel(ort_run_options_handle_t run_options, size_t level);
/**
* set a tag for the Run() calls using this.
*/
int EMSCRIPTEN_KEEPALIVE OrtRunOptionsSetRunTag(ort_run_options_handle_t run_options, const char* tag);
/**
* inference the model.
* @param session handle of the specified session
@ -106,5 +184,6 @@ int EMSCRIPTEN_KEEPALIVE OrtRun(ort_session_handle_t session,
size_t input_count,
const char** output_names,
size_t output_count,
ort_tensor_handle_t* outputs);
ort_tensor_handle_t* outputs,
ort_run_options_handle_t run_options);
};