mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-21 21:52:11 +00:00
e5ca3f3dcb
[//]: # (## Work In Progress. Feedbacks are welcome!) ### Description This PR adds a few properties, methods and factories to Tensor type to support IO-binding feature. This will allow user to create tensor from GPU/CPU bound data without a force transferring of data between CPU and GPU. This change is a way to resolve #15312 ### Change Summary 1. Add properties to `Tensor` type: a. `location`: indicating where the data is sitting. valid values are `cpu`, `cpu-pinned`, `texture`, `gpu-buffer`. b. `texture`: sit side to `data`, a readonly property of `WebGLTexture` type. available only when `location === 'texture'` c. `gpuBuffer`: sit side to `data`, a readonly property of `GPUBuffer` type. available only when `location === 'gpu-buffer'` 2. Add methods to `Tensor` type (usually dealing with inference outputs): - async function `getData()` allows user to download data from GPU to CPU manually. - function `dispose()` allows user to release GPU resources manually. 3. Add factories for creating `Tensor` instances: a. `fromTexture()` to create a WebGL texture bound tensor data b. `fromGpuBuffer()` to create a WebGPUBuffer bound tensor data c. `fromPinnedBuffer()` to create a tensor using a CPU pinned buffer ### Examples: create tensors from texture and pass to inference session as inputs ```js // when create session, specify we prefer 'image_output:0' to be stored on GPU as texture const session = await InferenceSession.create('./my_model.onnx', { executionProviders: [ 'webgl' ], preferredOutputLocation: { 'image_output:0': 'texture' } }); ... const myImageTexture = getTexture(); // user's function to get a texture const myFeeds = { input0: Tensor.fromTexture(myImageTexture, { width: 224, height: 224 }) }; // shape [1, 224, 224, 4], RGBA format. const results = await session.run(myFeeds); const myOutputTexture = results['image_output:0'].texture; ```
29 lines
1.2 KiB
TypeScript
29 lines
1.2 KiB
TypeScript
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
// Licensed under the MIT License.
|
|
|
|
/* eslint-disable @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports */
|
|
// We use "require" instead of "import" here because import statement must be put in top level. Our current code does
|
|
// not allow terser to tree-shaking code as expected because some codes are treated as having side effects.
|
|
// So we import code inside the if-clause to allow terser remove the code safely.
|
|
|
|
export * from 'onnxruntime-common';
|
|
import {registerBackend, env} from 'onnxruntime-common';
|
|
import {version} from './version';
|
|
|
|
if (!BUILD_DEFS.DISABLE_WEBGL) {
|
|
const onnxjsBackend = require('./backend-onnxjs').onnxjsBackend;
|
|
registerBackend('webgl', onnxjsBackend, -10);
|
|
}
|
|
|
|
if (!BUILD_DEFS.DISABLE_WASM) {
|
|
const wasmBackend = require('./backend-wasm').wasmBackend;
|
|
if (!BUILD_DEFS.DISABLE_WEBGPU && typeof navigator !== 'undefined' && navigator.gpu) {
|
|
registerBackend('webgpu', wasmBackend, 5);
|
|
}
|
|
registerBackend('cpu', wasmBackend, 10);
|
|
registerBackend('wasm', wasmBackend, 10);
|
|
registerBackend('xnnpack', wasmBackend, 9);
|
|
registerBackend('webnn', wasmBackend, 9);
|
|
}
|
|
|
|
Object.defineProperty(env.versions, 'web', {value: version, enumerable: true});
|