saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-05 04:17:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[js/web] support string tensor for wasm backend (#7891)

Browse source 
* [js/web] support string tensor for wasm backend

* disable v9/test_cast_STRING_to_FLOAT: test data is wrong

* add non-string check

* Update session-handler.ts

* Update session-handler.ts
This commit is contained in:
Yulong Wang 2021-06-03 00:44:50 -07:00 committed by GitHub
parent b854f2399d
commit 896f32ec09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 195 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
js/web/lib/onnxjs/tensor.ts
View file

@ -186,7 +186,7 @@ export class Tensor {
}
if (empty) {
cache = new Array<string>(size);
this.cache = new Array<string>(size);
}
} else {
if (cache !== undefined) {

2
js/web/lib/wasm/binding/ort-wasm.d.ts vendored
View file

@ -7,7 +7,7 @@ export interface OrtWasmModule extends EmscriptenModule {
stackRestore(stack: number): void;
stackAlloc(size: number): number;
UTF8ToString(offset: number): string;
UTF8ToString(offset: number, maxBytesToRead?: number): string;
lengthBytesUTF8(str: string): number;
stringToUTF8(str: string, offset: number, maxBytes: number): void;
//#endregion

13
js/web/lib/wasm/options-utils.ts
View file

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {getInstance} from './wasm-factory';
interface ExtraOptionsHandler {
(name: string, value: string): void;
}
@ -31,14 +29,3 @@ export const iterateExtraOptions =
}
});
};
export const allocWasmString = (data: string, allocs: number[]): number => {
const wasm = getInstance();
const dataLength = wasm.lengthBytesUTF8(data) + 1;
const dataOffset = wasm._malloc(dataLength);
wasm.stringToUTF8(data, dataOffset, dataLength);
allocs.push(dataOffset);
return dataOffset;
};

3
js/web/lib/wasm/run-options.ts
View file

@ -3,7 +3,8 @@
import {InferenceSession} from 'onnxruntime-common';
import {allocWasmString, iterateExtraOptions} from './options-utils';
import {iterateExtraOptions} from './options-utils';
import {allocWasmString} from './string-utils';
import {getInstance} from './wasm-factory';
export const setRunOptions = (options: InferenceSession.RunOptions): [number, number[]] => {

90
js/web/lib/wasm/session-handler.ts
View file

@ -6,6 +6,7 @@ import {env, InferenceSession, SessionHandler, Tensor, TypedTensor} from 'onnxru
import {setRunOptions} from './run-options';
import {setSessionOptions} from './session-options';
import {allocWasmString} from './string-utils';
import {getInstance} from './wasm-factory';
let ortInit: boolean;
@ -212,10 +213,6 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
if (index === -1) {
throw new Error(`invalid input '${name}'`);
}
if (tensor.type === 'string') {
// TODO: support string tensor
throw new TypeError('string tensor is not supported');
}
inputArray.push(tensor);
inputIndices.push(index);
});
@ -235,41 +232,55 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
const outputCount = outputIndices.length;
let runOptionsHandle = 0;
let allocs: number[] = [];
let runOptionsAllocs: number[] = [];
const inputValues: number[] = [];
const inputDataOffsets: number[] = [];
const inputAllocs: number[] = [];
try {
[runOptionsHandle, allocs] = setRunOptions(options);
[runOptionsHandle, runOptionsAllocs] = setRunOptions(options);
// create input tensors
for (let i = 0; i < inputCount; i++) {
const data = inputArray[i].data;
let dataOffset: number;
let dataByteLength: number;
if (Array.isArray(data)) {
// string tensor
throw new TypeError('string tensor is not supported');
} else {
const dataOffset = wasm._malloc(data.byteLength);
inputDataOffsets.push(dataOffset);
wasm.HEAPU8.set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), dataOffset);
const dims = inputArray[i].dims;
const stack = wasm.stackSave();
const dimsOffset = wasm.stackAlloc(4 * dims.length);
try {
let dimIndex = dimsOffset / 4;
dims.forEach(d => wasm.HEAP32[dimIndex++] = d);
const tensor = wasm._OrtCreateTensor(
tensorDataTypeStringToEnum(inputArray[i].type), dataOffset, data.byteLength, dimsOffset, dims.length);
if (tensor === 0) {
throw new Error('Can\'t create a tensor');
dataByteLength = 4 * data.length;
dataOffset = wasm._malloc(dataByteLength);
inputAllocs.push(dataOffset);
let dataIndex = dataOffset / 4;
for (let i = 0; i < data.length; i++) {
if (typeof data[i] !== 'string') {
throw new TypeError(`tensor data at index ${i} is not a string`);
}
inputValues.push(tensor);
} finally {
wasm.stackRestore(stack);
wasm.HEAPU32[dataIndex++] = allocWasmString(data[i], inputAllocs);
}
} else {
dataByteLength = data.byteLength;
dataOffset = wasm._malloc(dataByteLength);
inputAllocs.push(dataOffset);
wasm.HEAPU8.set(new Uint8Array(data.buffer, data.byteOffset, dataByteLength), dataOffset);
}
const dims = inputArray[i].dims;
const stack = wasm.stackSave();
const dimsOffset = wasm.stackAlloc(4 * dims.length);
try {
let dimIndex = dimsOffset / 4;
dims.forEach(d => wasm.HEAP32[dimIndex++] = d);
const tensor = wasm._OrtCreateTensor(
tensorDataTypeStringToEnum(inputArray[i].type), dataOffset, dataByteLength, dimsOffset, dims.length);
if (tensor === 0) {
throw new Error('Can\'t create a tensor');
}
inputValues.push(tensor);
} finally {
wasm.stackRestore(stack);
}
}
@ -307,6 +318,8 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
const beforeGetTensorDataStack = wasm.stackSave();
// stack allocate 4 pointer value
const tensorDataOffset = wasm.stackAlloc(4 * 4);
let type: Tensor.Type|undefined, dataOffset = 0;
try {
errorCode = wasm._OrtGetTensorData(
tensor, tensorDataOffset, tensorDataOffset + 4, tensorDataOffset + 8, tensorDataOffset + 12);
@ -315,7 +328,7 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
}
let tensorDataIndex = tensorDataOffset / 4;
const dataType = wasm.HEAPU32[tensorDataIndex++];
const dataOffset: number = wasm.HEAPU32[tensorDataIndex++];
dataOffset = wasm.HEAPU32[tensorDataIndex++];
const dimsOffset = wasm.HEAPU32[tensorDataIndex++];
const dimsLength = wasm.HEAPU32[tensorDataIndex++];
const dims = [];
@ -324,13 +337,19 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
}
wasm._OrtFree(dimsOffset);
const type = tensorDataTypeEnumToString(dataType);
const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b);
type = tensorDataTypeEnumToString(dataType);
if (type === 'string') {
// string tensor
throw new TypeError('string tensor is not supported');
const stringData: string[] = [];
let dataIndex = dataOffset / 4;
for (let i = 0; i < size; i++) {
const offset = wasm.HEAPU32[dataIndex++];
const maxBytesToRead = i === size - 1 ? undefined : wasm.HEAPU32[dataIndex] - offset;
stringData.push(wasm.UTF8ToString(offset, maxBytesToRead));
}
output[this.outputNames[outputIndices[i]]] = new Tensor('string', stringData, dims);
} else {
const typedArray = numericTensorTypeToTypedArray(type);
const size = dims.length === 0 ? 1 : dims.reduce((a, b) => a * b);
const t = new Tensor(type, new typedArray(size), dims) as TypedTensor<Exclude<Tensor.Type, 'string'>>;
new Uint8Array(t.data.buffer, t.data.byteOffset, t.data.byteLength)
.set(wasm.HEAPU8.subarray(dataOffset, dataOffset + t.data.byteLength));
@ -338,6 +357,9 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
}
} finally {
wasm.stackRestore(beforeGetTensorDataStack);
if (type === 'string' && dataOffset) {
wasm._free(dataOffset);
}
wasm._OrtReleaseTensor(tensor);
}
}
@ -353,10 +375,10 @@ export class OnnxruntimeWebAssemblySessionHandler implements SessionHandler {
}
} finally {
inputValues.forEach(wasm._OrtReleaseTensor);
inputDataOffsets.forEach(wasm._free);
inputAllocs.forEach(wasm._free);
wasm._OrtReleaseRunOptions(runOptionsHandle);
allocs.forEach(wasm._free);
runOptionsAllocs.forEach(wasm._free);
}
}

3
js/web/lib/wasm/session-options.ts
View file

@ -3,7 +3,8 @@
import {InferenceSession} from 'onnxruntime-common';
import {allocWasmString, iterateExtraOptions} from './options-utils';
import {iterateExtraOptions} from './options-utils';
import {allocWasmString} from './string-utils';
import {getInstance} from './wasm-factory';
const getGraphOptimzationLevel = (graphOptimizationLevel: string|unknown): number => {

15
js/web/lib/wasm/string-utils.ts Normal file
View file

@ -0,0 +1,15 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import {getInstance} from './wasm-factory';
export const allocWasmString = (data: string, allocs: number[]): number => {
const wasm = getInstance();
const dataLength = wasm.lengthBytesUTF8(data) + 1;
const dataOffset = wasm._malloc(dataLength);
wasm.stringToUTF8(data, dataOffset, dataLength);
allocs.push(dataOffset);
return dataOffset;
};

17
js/web/test/test-runner.ts
View file

@ -382,7 +382,12 @@ export class TensorResultValidator {
}
for (let i = actual.length - 1; i >= 0; i--) {
const a = actual[i], b = Math.max(Math.min(expected[i], this.maxFloatValue), -this.maxFloatValue);
const a = actual[i];
let b = expected[i];
if (a === b) {
continue; // exact the same value, treat as equal
}
// check for NaN
//
@ -394,6 +399,16 @@ export class TensorResultValidator {
return false; // one is NaN and the other is not
}
// check for Infinity
//
if (!Number.isFinite(a) || !Number.isFinite(b)) {
Logger.error('Validator', `a or b is Infinity -- index:${i}: actual=${actual[i]},expected=${expected[i]}`);
return false; // at least one is Infinity and the other is not or their sign is different
}
// normalize value of b
b = Math.max(Math.min(expected[i], this.maxFloatValue), -this.maxFloatValue);
// Comparing 2 float numbers: (Suppose a >= b)
//
// if ( a - b < ABSOLUTE_ERROR || 1.0 < a / b < RELATIVE_ERROR)

1
js/web/test/test-suite-whitelist.jsonc
View file

@ -315,6 +315,7 @@
"test_basic_conv_without_padding",
"test_batchnorm_epsilon",
"test_batchnorm_example",
"v{10,11,12}/test_cast_STRING_to_FLOAT",
"v{7,8,9,10}/test_clip_splitbounds",
"v{7,8,9,10}/test_clip_outbounds",
"v{7,8,9,10}/test_clip_inbounds",

134
onnxruntime/wasm/api.cc
View file

@ -26,20 +26,20 @@ OrtErrorCode CheckStatus(OrtStatusPtr status) {
#define CHECK_STATUS(ORT_API_NAME, ...) \
CheckStatus(Ort::GetApi().ORT_API_NAME(__VA_ARGS__))
#define RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \
do { \
int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \
if (error_code != ORT_OK) { \
return error_code; \
} \
#define RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \
do { \
int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \
if (error_code != ORT_OK) { \
return error_code; \
} \
} while (false)
// TODO: This macro can be removed when we changed all APIs to return a status code.
#define RETURN_NULLPTR_IF_ERROR(ORT_API_NAME, ...) \
do { \
if (CHECK_STATUS(ORT_API_NAME, __VA_ARGS__) != ORT_OK) { \
return nullptr; \
} \
#define RETURN_NULLPTR_IF_ERROR(ORT_API_NAME, ...) \
do { \
if (CHECK_STATUS(ORT_API_NAME, __VA_ARGS__) != ORT_OK) { \
return nullptr; \
} \
} while (false)
int OrtInit(int num_threads, int logging_level) {
@ -107,8 +107,8 @@ OrtSessionOptions* OrtCreateSessionOptions(size_t graph_optimization_level,
}
int OrtAddSessionConfigEntry(OrtSessionOptions* session_options,
const char* config_key,
const char* config_value) {
const char* config_key,
const char* config_value) {
return CHECK_STATUS(AddSessionConfigEntry, session_options, config_key, config_value);
}
@ -132,7 +132,8 @@ OrtSession* OrtCreateSession(void* data, size_t data_length, OrtSessionOptions*
OrtSession* session = nullptr;
return (CHECK_STATUS(CreateSessionFromArray, g_env, data, data_length, session_options, &session) == ORT_OK)
? session : nullptr;
? session
: nullptr;
}
void OrtReleaseSession(OrtSession* session) {
@ -155,7 +156,8 @@ char* OrtGetInputName(OrtSession* session, size_t index) {
char* input_name = nullptr;
return (CHECK_STATUS(SessionGetInputName, session, index, allocator, &input_name) == ORT_OK)
? input_name : nullptr;
? input_name
: nullptr;
}
char* OrtGetOutputName(OrtSession* session, size_t index) {
@ -164,7 +166,8 @@ char* OrtGetOutputName(OrtSession* session, size_t index) {
char* output_name = nullptr;
return (CHECK_STATUS(SessionGetOutputName, session, index, allocator, &output_name) == ORT_OK)
? output_name : nullptr;
? output_name
: nullptr;
}
void OrtFree(void* ptr) {
@ -180,36 +183,55 @@ OrtValue* OrtCreateTensor(int data_type, void* data, size_t data_length, size_t*
shapes[i] = dims[i];
}
OrtMemoryInfo* memoryInfo = nullptr;
RETURN_NULLPTR_IF_ERROR(CreateCpuMemoryInfo, OrtDeviceAllocator, OrtMemTypeDefault, &memoryInfo);
if (data_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) {
OrtAllocator* allocator = nullptr;
RETURN_NULLPTR_IF_ERROR(GetAllocatorWithDefaultOptions, &allocator);
OrtValue* value = nullptr;
int error_code = CHECK_STATUS(CreateTensorWithDataAsOrtValue, memoryInfo, data, data_length,
dims_length > 0 ? shapes.data() : nullptr, dims_length,
static_cast<ONNXTensorElementDataType>(data_type), &value);
OrtValue* value = nullptr;
RETURN_NULLPTR_IF_ERROR(CreateTensorAsOrtValue, allocator,
dims_length > 0 ? shapes.data() : nullptr, dims_length,
ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, &value);
Ort::GetApi().ReleaseMemoryInfo(memoryInfo);
return (error_code == ORT_OK) ? value : nullptr;
const char* const* strings = reinterpret_cast<const char* const*>(data);
RETURN_NULLPTR_IF_ERROR(FillStringTensor, value, strings, data_length / sizeof(const char*));
return value;
} else {
OrtMemoryInfo* memoryInfo = nullptr;
RETURN_NULLPTR_IF_ERROR(CreateCpuMemoryInfo, OrtDeviceAllocator, OrtMemTypeDefault, &memoryInfo);
OrtValue* value = nullptr;
int error_code = CHECK_STATUS(CreateTensorWithDataAsOrtValue, memoryInfo, data, data_length,
dims_length > 0 ? shapes.data() : nullptr, dims_length,
static_cast<ONNXTensorElementDataType>(data_type), &value);
Ort::GetApi().ReleaseMemoryInfo(memoryInfo);
return (error_code == ORT_OK) ? value : nullptr;
}
}
int OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dims, size_t* dims_length) {
#define RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \
do { \
int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \
if (error_code != ORT_OK) { \
if (info != nullptr) { \
Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); \
} \
if (allocator != nullptr && p_dims != nullptr) { \
allocator->Free(allocator, p_dims); \
} \
return error_code; \
} \
#define RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(ORT_API_NAME, ...) \
do { \
int error_code = CHECK_STATUS(ORT_API_NAME, __VA_ARGS__); \
if (error_code != ORT_OK) { \
if (info != nullptr) { \
Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info); \
} \
if (allocator != nullptr && p_dims != nullptr) { \
allocator->Free(allocator, p_dims); \
} \
if (allocator != nullptr && p_string_data != nullptr) { \
allocator->Free(allocator, p_string_data); \
} \
return error_code; \
} \
} while (false)
OrtTensorTypeAndShapeInfo* info = nullptr;
OrtAllocator* allocator = nullptr;
size_t* p_dims = nullptr;
void* p_string_data = nullptr;
RETURN_ERROR_CODE_IF_ERROR(GetTensorTypeAndShape, tensor, &info);
@ -233,6 +255,42 @@ int OrtGetTensorData(OrtValue* tensor, int* data_type, void** data, size_t** dim
}
*dims = p_dims;
if (type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) {
size_t num_elements;
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetTensorShapeElementCount, info, &num_elements);
// NOTE: ORT C-API does not expose an interface for users to get string raw data directly. There is always a copy.
// we can use the tensor raw data because it is type of "std::string *", which is very starightforward to
// implement and can also save memory usage. However, this approach depends on the Tensor's implementation
// details. So we have to copy the string content here.
size_t string_data_length;
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetStringTensorDataLength, tensor, &string_data_length);
// The buffer contains following data:
// - a sequence of pointers to (const char*), size = num_elements * sizeof(const char*).
// - followed by a raw buffer to store string content, size = string_data_length + 1.
static_assert(sizeof(const char*) == sizeof(size_t), "size of a pointer and a size_t value should be the same.");
size_t string_data_offset = num_elements * sizeof(const char*);
size_t buf_size = string_data_offset + string_data_length;
p_string_data = allocator->Alloc(allocator, buf_size + 1);
void* p_string_content = reinterpret_cast<char*>(p_string_data) + string_data_offset;
size_t* p_offsets = reinterpret_cast<size_t*>(p_string_data);
RELEASE_AND_RETURN_ERROR_CODE_IF_ERROR(GetStringTensorContent, tensor, p_string_content, string_data_length, p_offsets, num_elements);
// replace offsets by pointers
const char** p_c_strs = reinterpret_cast<const char**>(p_offsets);
for (size_t i = 0; i < num_elements; i++) {
p_c_strs[i] = reinterpret_cast<const char*>(p_string_content) + p_offsets[i];
}
// put null at the last char
reinterpret_cast<char*>(p_string_data)[buf_size] = '\0';
*data = p_string_data;
}
Ort::GetApi().ReleaseTensorTypeAndShapeInfo(info);
return ORT_OK;
}
@ -267,8 +325,8 @@ OrtRunOptions* OrtCreateRunOptions(size_t log_severity_level,
}
int OrtAddRunConfigEntry(OrtRunOptions* run_options,
const char* config_key,
const char* config_value) {
const char* config_key,
const char* config_value) {
return CHECK_STATUS(AddRunConfigEntry, run_options, config_key, config_value);
}

9
onnxruntime/wasm/api.h
View file

@ -118,8 +118,8 @@ void EMSCRIPTEN_KEEPALIVE OrtFree(void* ptr);
/**
* create an instance of ORT tensor.
* @param data_type data type defined in enum ONNXTensorElementDataType.
* @param data a pointer to the tensor data.
* @param data_length size of the tensor data in bytes.
* @param data for numeric tensor: a pointer to the tensor data buffer. for string tensor: a pointer to a C-Style null terminated string array.
* @param data_length size of the buffer 'data' in bytes.
* @param dims a pointer to an array of dims. the array should contain (dims_length) element(s).
* @param dims_length the length of the tensor's dimension
* @returns a handle of the tensor.
@ -130,10 +130,11 @@ ort_tensor_handle_t EMSCRIPTEN_KEEPALIVE OrtCreateTensor(int data_type, void* da
* get type, shape info and data of the specified tensor.
* @param tensor handle of the tensor.
* @param data_type [out] specify the memory to write data type
* @param data [out] specify the memory to write the tensor data
* @param data [out] specify the memory to write the tensor data. for string tensor: an array of C-Style null terminated string.
* @param dims [out] specify the memory to write address of the buffer containing value of each dimension.
* @param dims_length [out] specify the memory to write dims length
* @remarks a temporary buffer 'dims' is allocated during the call. Caller must release the buffer after use by calling OrtFree().
* @remarks following temporary buffers are allocated during the call. Caller must release the buffers after use by calling OrtFree():
* 'dims' (for all types of tensor), 'data' (only for string tensor)
*/
int EMSCRIPTEN_KEEPALIVE OrtGetTensorData(ort_tensor_handle_t tensor, int* data_type, void** data, size_t** dims, size_t* dims_length);