2021-04-27 07:04:25 +00:00
|
|
|
{
|
2021-05-15 01:15:38 +00:00
|
|
|
"cpu": {
|
|
|
|
|
"onnx": [],
|
|
|
|
|
"node": [],
|
|
|
|
|
"ops": []
|
|
|
|
|
},
|
2021-04-27 07:04:25 +00:00
|
|
|
"webgl": {
|
2023-08-01 08:10:55 +00:00
|
|
|
"onnx": ["resnet50", "squeezenet", "tiny_yolov2", "emotion_ferplus"],
|
2021-04-27 07:04:25 +00:00
|
|
|
"node": [
|
|
|
|
|
"test_abs",
|
|
|
|
|
"test_acos_example",
|
|
|
|
|
"test_acos",
|
|
|
|
|
"test_add_bcast",
|
|
|
|
|
"test_add",
|
|
|
|
|
"test_and_bcast3v1d",
|
|
|
|
|
"test_and_bcast3v2d",
|
|
|
|
|
"test_and_bcast4v2d",
|
|
|
|
|
"test_and_bcast4v3d",
|
|
|
|
|
"test_and_bcast4v4d",
|
|
|
|
|
"test_and2d",
|
|
|
|
|
"test_and3d",
|
|
|
|
|
"test_and4d",
|
|
|
|
|
"test_asin_example",
|
|
|
|
|
"test_asin",
|
|
|
|
|
"test_atan_example",
|
|
|
|
|
"test_atan",
|
|
|
|
|
"test_averagepool_1d_default",
|
|
|
|
|
"test_averagepool_2d_default",
|
2022-07-21 05:01:08 +00:00
|
|
|
//"opset12/test_averagepool_2d_pads", // TODO: fix avgpool and maxpool on VM
|
|
|
|
|
// "opset12/test_averagepool_2d_precomputed_pads",
|
|
|
|
|
// "opset12/test_averagepool_2d_precomputed_same_upper",
|
|
|
|
|
// "opset12/test_averagepool_2d_precomputed_strides",
|
|
|
|
|
// "opset12/test_averagepool_2d_same_upper",
|
|
|
|
|
// "opset12/test_averagepool_2d_same_lower",
|
|
|
|
|
// "opset12/test_averagepool_2d_strides",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_averagepool_3d_default",
|
|
|
|
|
"test_basic_conv_with_padding",
|
|
|
|
|
"test_basic_conv_without_padding",
|
|
|
|
|
"test_batchnorm_epsilon",
|
|
|
|
|
"test_batchnorm_example",
|
2021-10-14 23:29:37 +00:00
|
|
|
"test_cast_DOUBLE_to_FLOAT",
|
|
|
|
|
"test_cast_FLOAT_to_DOUBLE",
|
2022-07-21 05:01:08 +00:00
|
|
|
"opset{7,8,9,10}/test_clip_splitbounds",
|
|
|
|
|
"opset{7,8,9,10}/test_clip_outbounds",
|
|
|
|
|
"opset{7,8,9,10}/test_clip_inbounds",
|
|
|
|
|
"opset{7,8,9,10}/test_clip_example",
|
|
|
|
|
"opset{7,8,9,10}/test_clip_default_min",
|
|
|
|
|
"opset{7,8,9,10}/test_clip_default_max",
|
|
|
|
|
"opset{7,8,9,10}/test_clip_default_inbounds",
|
|
|
|
|
"opset{7,8,9,10}/test_clip",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_concat_1d_axis_0",
|
|
|
|
|
"test_concat_2d_axis_0",
|
|
|
|
|
"test_concat_2d_axis_1",
|
|
|
|
|
"test_concat_3d_axis_0",
|
|
|
|
|
"test_concat_3d_axis_1",
|
|
|
|
|
"test_concat_3d_axis_2",
|
|
|
|
|
"test_conv_with_strides_and_asymmetric_padding",
|
|
|
|
|
"test_conv_with_strides_no_padding",
|
|
|
|
|
"test_conv_with_strides_padding",
|
2022-07-27 20:57:12 +00:00
|
|
|
"test_convtranspose",
|
|
|
|
|
"test_convtranspose_pad",
|
|
|
|
|
"test_convtranspose_pads",
|
|
|
|
|
// TODO: add this when test-case file in opset v8 is fixed (i.e. output_shape has 2 dims)
|
|
|
|
|
// Might have to rewrite git history for that...
|
|
|
|
|
// "test_convtranspose_output_shape",
|
|
|
|
|
"test_convtranspose_kernel_shape",
|
|
|
|
|
"test_convtranspose_dilations",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_constant",
|
|
|
|
|
"test_cos_example",
|
|
|
|
|
"test_cos",
|
|
|
|
|
"test_div_bcast",
|
|
|
|
|
"test_div_example",
|
|
|
|
|
"test_div",
|
|
|
|
|
"test_dropout_default",
|
|
|
|
|
"test_dropout_random",
|
2021-05-26 00:43:43 +00:00
|
|
|
"test_depthtospace_crd_mode",
|
|
|
|
|
"test_depthtospace_crd_mode_example",
|
|
|
|
|
"test_depthtospace_dcr_mode",
|
|
|
|
|
"test_depthtospace_example",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_elu_example",
|
|
|
|
|
"test_elu",
|
|
|
|
|
"test_elu_default",
|
|
|
|
|
"test_flatten_axis0",
|
|
|
|
|
"test_flatten_axis1",
|
|
|
|
|
"test_flatten_axis2",
|
|
|
|
|
"test_flatten_axis3",
|
|
|
|
|
"test_flatten_default_axis",
|
|
|
|
|
"test_gather_0",
|
|
|
|
|
"test_gather_1",
|
|
|
|
|
"test_gemm_nobroadcast",
|
|
|
|
|
"test_gemm_broadcast",
|
|
|
|
|
"test_globalaveragepool_precomputed",
|
|
|
|
|
"test_globalaveragepool",
|
|
|
|
|
"test_globalmaxpool_precomputed",
|
|
|
|
|
"test_globalmaxpool",
|
|
|
|
|
"test_greater_bcast",
|
|
|
|
|
"test_greater",
|
|
|
|
|
"test_instancenorm_epsilon",
|
|
|
|
|
"test_instancenorm_example",
|
|
|
|
|
"test_less_bcast",
|
|
|
|
|
"test_less",
|
|
|
|
|
"test_equal_bcast",
|
|
|
|
|
"test_equal",
|
|
|
|
|
"test_identity",
|
|
|
|
|
"test_leakyrelu_default",
|
|
|
|
|
"test_leakyrelu_example",
|
|
|
|
|
"test_leakyrelu",
|
2023-02-01 19:51:10 +00:00
|
|
|
"test_lrn_default",
|
|
|
|
|
"test_lrn",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_matmul_2d",
|
|
|
|
|
"test_matmul_3d",
|
|
|
|
|
"test_matmul_4d",
|
|
|
|
|
"test_maxpool_1d_default",
|
|
|
|
|
"test_maxpool_2d_default",
|
2022-07-21 05:01:08 +00:00
|
|
|
// "opset12/test_maxpool_2d_pads",
|
|
|
|
|
// "opset12/test_maxpool_2d_precomputed_pads",
|
|
|
|
|
// "opset12/test_maxpool_2d_precomputed_same_upper",
|
|
|
|
|
// "opset12/test_maxpool_2d_precomputed_strides",
|
|
|
|
|
// "opset12/test_maxpool_2d_same_lower",
|
|
|
|
|
// "opset12/test_maxpool_2d_same_upper",
|
|
|
|
|
// "opset12/test_maxpool_2d_strides",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_maxpool_3d_default",
|
|
|
|
|
"test_mul_bcast",
|
|
|
|
|
"test_mul_example",
|
|
|
|
|
"test_mul",
|
|
|
|
|
"test_neg",
|
|
|
|
|
"test_neg_example",
|
|
|
|
|
"test_not_2d",
|
|
|
|
|
"test_not_3d",
|
|
|
|
|
"test_not_4d",
|
|
|
|
|
"test_or_bcast3v1d",
|
|
|
|
|
"test_or_bcast3v2d",
|
|
|
|
|
"test_or_bcast4v2d",
|
|
|
|
|
"test_or_bcast4v3d",
|
|
|
|
|
"test_or_bcast4v4d",
|
|
|
|
|
"test_prelu_broadcast",
|
|
|
|
|
"test_prelu_example",
|
|
|
|
|
"test_relu",
|
|
|
|
|
"test_reshape_extended_dims",
|
|
|
|
|
"test_reshape_negative_dim",
|
|
|
|
|
"test_reshape_one_dim",
|
|
|
|
|
"test_reshape_reduced_dims",
|
|
|
|
|
"test_reshape_reordered_dims",
|
|
|
|
|
"test_sigmoid",
|
|
|
|
|
"test_sigmoid_example",
|
|
|
|
|
"test_sin_example",
|
|
|
|
|
"test_sin",
|
2021-10-27 06:58:50 +00:00
|
|
|
"test_softmax_axis_0",
|
|
|
|
|
"test_softmax_axis_1",
|
|
|
|
|
"test_softmax_axis_2",
|
|
|
|
|
"test_softmax_default_axis",
|
|
|
|
|
"test_softmax_example",
|
2021-04-27 07:04:25 +00:00
|
|
|
{
|
2021-10-27 06:58:50 +00:00
|
|
|
"name": "test_softmax_large_number",
|
2023-08-03 21:20:20 +00:00
|
|
|
"platformCondition": "^((?!iOS).)*$" // does NOT contains 'iOS': large number cannot be handled in a half_float environment
|
2021-04-27 07:04:25 +00:00
|
|
|
},
|
|
|
|
|
"test_sub_bcast",
|
|
|
|
|
"test_sub_example",
|
|
|
|
|
"test_sub",
|
|
|
|
|
"test_sum_example",
|
|
|
|
|
"test_sum_one_input",
|
|
|
|
|
"test_sum_two_inputs",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_log_sum_asc_axes",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_log_sum_default",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_log_sum_desc_axes",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_max_default_axes_keepdim_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_max_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_max_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_max_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_max_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_max_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_mean_default_axes_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_mean_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_mean_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_mean_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_mean_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_mean_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_min_default_axes_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_min_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_min_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_min_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_min_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_min_keepdims_random",
|
2021-04-27 07:04:25 +00:00
|
|
|
{
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"name": "opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_prod_default_axes_keepdims_example",
|
2023-08-03 21:20:20 +00:00
|
|
|
"platformCondition": "^((?!iOS).)*$" // does NOT contains 'iOS': large number cannot be handled in a half_float environment
|
2021-04-27 07:04:25 +00:00
|
|
|
},
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_prod_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_prod_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_prod_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_prod_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12,13,14,15,16,17}/test_reduce_prod_keepdims_random",
|
2022-07-21 05:01:08 +00:00
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_default_axes_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_square_default_axes_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_square_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_square_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_square_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_square_keepdims_example",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_reduce_sum_square_keepdims_random",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_split_variable_parts_default_axis",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_split_variable_parts_1d",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_split_variable_parts_2d",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_split_equal_parts_default_axis",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_split_equal_parts_1d",
|
|
|
|
|
"opset{7,8,9,10,11,12}/test_split_equal_parts_2d",
|
|
|
|
|
"opset{7,8,9}/test_slice",
|
|
|
|
|
"opset{7,8,9}/test_slice_default_axes",
|
|
|
|
|
"opset{7,8,9}/test_slice_end_out_of_bounds",
|
|
|
|
|
"opset{7,8,9}/test_slice_neg",
|
2021-04-27 07:04:25 +00:00
|
|
|
// "test_slice_start_out_of_bounds", // tensor shape of 0
|
|
|
|
|
"test_squeeze",
|
|
|
|
|
"test_tan_example",
|
2022-03-30 04:47:52 +00:00
|
|
|
//"test_tan", // incorrect result in Chrome v100... investigating
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_tanh_example",
|
|
|
|
|
"test_tanh",
|
|
|
|
|
"test_tile",
|
|
|
|
|
"test_tile_precomputed",
|
|
|
|
|
"test_transpose_all_permutations_0",
|
|
|
|
|
"test_transpose_all_permutations_1",
|
|
|
|
|
"test_transpose_all_permutations_2",
|
|
|
|
|
"test_transpose_all_permutations_3",
|
|
|
|
|
"test_transpose_all_permutations_4",
|
|
|
|
|
"test_transpose_all_permutations_5",
|
|
|
|
|
"test_transpose_default",
|
|
|
|
|
"test_unsqueeze",
|
|
|
|
|
"test_xor_bcast3v1d",
|
|
|
|
|
"test_xor_bcast3v2d",
|
|
|
|
|
"test_xor_bcast4v2d",
|
|
|
|
|
"test_xor_bcast4v3d",
|
|
|
|
|
"test_xor_bcast4v4d",
|
|
|
|
|
"test_xor2d",
|
|
|
|
|
"test_xor3d",
|
|
|
|
|
"test_xor4d"
|
|
|
|
|
],
|
|
|
|
|
"ops": [
|
|
|
|
|
"abs.jsonc",
|
|
|
|
|
"acos.jsonc",
|
|
|
|
|
"add.jsonc",
|
|
|
|
|
"and.jsonc",
|
|
|
|
|
"asin.jsonc",
|
|
|
|
|
"ceil.jsonc",
|
2021-08-12 19:30:49 +00:00
|
|
|
"concat.jsonc",
|
2021-04-27 07:04:25 +00:00
|
|
|
"conv.jsonc",
|
|
|
|
|
"cos.jsonc",
|
|
|
|
|
"div.jsonc",
|
2021-05-26 00:43:43 +00:00
|
|
|
"depth-to-space.jsonc",
|
2021-04-27 07:04:25 +00:00
|
|
|
"equal.jsonc",
|
|
|
|
|
"exp.jsonc",
|
|
|
|
|
"floor.jsonc",
|
|
|
|
|
"global-average-pool.jsonc",
|
|
|
|
|
"gemm.jsonc",
|
|
|
|
|
"greater.jsonc",
|
|
|
|
|
//"identity.jsonc",
|
|
|
|
|
"image-scaler.jsonc",
|
[js/webgpu] Optimize InstanceNormalization (#17491)
### Description
<!-- Describe your changes. -->
In previous implementation, there are two loops to iterate H * W
elements to calculate the `mean` and `squaredNorm` value in one thread,
meanwhile it outputs H * W elements in one thread. That results it's
very very slow when H * W is a large value. And usually, H * W does be a
large value in a model. For example, in the `candy-8` model, the shapes
of [H, W] are [224,224], [112,112], [56,56] for `InstanceNormalization`
op. And in my ADL, `[1,224,224,32]` consumes 17 ms. See below:
```
[profiling] kernel "23848328|[InstanceNormalization] 23848328" input[0]: [1,224,224,32] | float32, input[1]: [32] | float32, input[2]: [32] | float32, output[0]: [1,224,224,32] | float32, execution time: 17007914 ns
```
In this PR, it uses workgroup memory to optimize the original algorithm.
The advantage is that it can parallelly utilize the 64 (workgroupSize)
threads in one workgroup to calculate `mean` and `squaredNorm` value.
Meanwhile, it only outputs `H * W / workgroupSize` outputs for one
thread, which greatly reduces the overhead for one thread. With this
optimization, `[1,224,224,32]` becomes 3 ms and the main overhead is the
extra two `transpose`. The `createInstanceNormProgramInfo` only needs
`0.64` ms. See below:
```
[profiling] kernel "23003600|[InstanceNormalization] 23003600" input[0]: [1,224,224,32] | float32, output[0]: [1,32,224,224] | float32, execution time: 1543792 ns
program-manager.ts:115
[profiling] kernel "23003600|[InstanceNormalization] 23003600" input[0]: [1,32,224,224] | float32, input[1]: [32] | float32, input[2]: [32] | float32, output[0]: [1,32,224,224] | float32, execution time: 642652 ns
program-manager.ts:115
[profiling] kernel "23003600|[InstanceNormalization] 23003600" input[0]: [1,32,224,224] | float32, output[0]: [1,224,224,32] | float32, execution time: 991608 ns
```
This PR currently only applies the new algorithm to NCHW format. For
NHWC format, one way is to transpose the input so that it can use the
new algorithm. But the disadvantage is that 2 extra transpose are added.
@dakenf also gives another way to optimize NHWC. Details see
[here](https://github.com/microsoft/onnxruntime/blob/d45a96616da9843b037210f2d48d6b4e5bdae5c6/js/web/lib/wasm/jsep/webgpu/ops/instance-norm.ts).
I checked @dakenf's method. The perf is similar with transpose +
optimized NCHW. But on different GPUs, one is a little better than
another or vice versa. So I prefer this PR only does the NCHW part.
@dakenf can submit his optimization on NHWC.
2023-09-15 00:03:18 +00:00
|
|
|
"instance-norm.jsonc",
|
2021-04-27 07:04:25 +00:00
|
|
|
"less.jsonc",
|
|
|
|
|
"log.jsonc",
|
|
|
|
|
"matmul.jsonc",
|
|
|
|
|
"mul.jsonc",
|
|
|
|
|
"neg.jsonc",
|
|
|
|
|
"not.jsonc",
|
|
|
|
|
"or.jsonc",
|
|
|
|
|
"leaky-relu.jsonc",
|
|
|
|
|
"reduce-min.jsonc",
|
|
|
|
|
"relu.jsonc",
|
|
|
|
|
"pad.jsonc",
|
|
|
|
|
"pad-big.jsonc",
|
|
|
|
|
"pow.jsonc",
|
|
|
|
|
"pow-big-number.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"reshape-int32.jsonc",
|
2021-04-27 07:04:25 +00:00
|
|
|
"softmax.jsonc",
|
|
|
|
|
"sin.jsonc",
|
|
|
|
|
"split.jsonc",
|
|
|
|
|
"sqrt.jsonc",
|
|
|
|
|
"sub.jsonc",
|
|
|
|
|
"tan.jsonc",
|
|
|
|
|
"transpose.jsonc",
|
|
|
|
|
"xor.jsonc"
|
|
|
|
|
]
|
|
|
|
|
},
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"webgpu": {
|
|
|
|
|
"onnx": [],
|
|
|
|
|
"node": [
|
|
|
|
|
"test_abs",
|
|
|
|
|
"test_acos_example",
|
|
|
|
|
"test_acos",
|
|
|
|
|
"test_acosh_example",
|
|
|
|
|
"test_acosh",
|
|
|
|
|
// // "test_adagrad_multiple",
|
|
|
|
|
// // "test_adagrad",
|
|
|
|
|
// // "test_adam_multiple",
|
|
|
|
|
// // "test_adam",
|
|
|
|
|
"test_add_bcast",
|
|
|
|
|
// "test_add_uint8",
|
|
|
|
|
"test_add",
|
|
|
|
|
// "test_and_bcast3v1d",
|
|
|
|
|
// "test_and_bcast3v2d",
|
|
|
|
|
// "test_and_bcast4v2d",
|
|
|
|
|
// "test_and_bcast4v3d",
|
|
|
|
|
// "test_and_bcast4v4d",
|
|
|
|
|
// "test_and2d",
|
|
|
|
|
// "test_and3d",
|
|
|
|
|
// "test_and4d",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_default_axis_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_default_axis_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_default_axis_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_default_axis_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_keepdims_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_keepdims_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_keepdims_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_keepdims_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_negative_axis_keepdims_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_negative_axis_keepdims_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_negative_axis_keepdims_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_negative_axis_keepdims_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_no_keepdims_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_no_keepdims_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmax_no_keepdims_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmax_no_keepdims_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_default_axis_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_default_axis_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_default_axis_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_default_axis_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_keepdims_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_keepdims_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_keepdims_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_keepdims_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_negative_axis_keepdims_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_negative_axis_keepdims_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_negative_axis_keepdims_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_negative_axis_keepdims_random",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_no_keepdims_example_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_no_keepdims_example",
|
2023-08-04 19:59:36 +00:00
|
|
|
"test_argmin_no_keepdims_random_select_last_index",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_argmin_no_keepdims_random",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_asin_example",
|
|
|
|
|
"test_asin",
|
|
|
|
|
"test_asinh_example",
|
|
|
|
|
"test_asinh",
|
|
|
|
|
"test_atan_example",
|
|
|
|
|
"test_atan",
|
|
|
|
|
"test_atanh_example",
|
|
|
|
|
"test_atanh",
|
|
|
|
|
// "test_averagepool_1d_default",
|
|
|
|
|
// "test_averagepool_2d_ceil",
|
|
|
|
|
"test_averagepool_2d_default",
|
|
|
|
|
"test_averagepool_2d_pads_count_include_pad",
|
|
|
|
|
"test_averagepool_2d_pads",
|
|
|
|
|
"test_averagepool_2d_precomputed_pads_count_include_pad",
|
|
|
|
|
"test_averagepool_2d_precomputed_pads",
|
|
|
|
|
"test_averagepool_2d_precomputed_same_upper",
|
|
|
|
|
"test_averagepool_2d_precomputed_strides",
|
|
|
|
|
"test_averagepool_2d_same_lower",
|
|
|
|
|
"test_averagepool_2d_same_upper",
|
|
|
|
|
"test_averagepool_2d_strides",
|
|
|
|
|
// "test_averagepool_3d_default",
|
|
|
|
|
"test_basic_conv_with_padding",
|
|
|
|
|
"test_basic_conv_without_padding",
|
|
|
|
|
// "test_basic_convinteger",
|
2023-08-12 15:53:10 +00:00
|
|
|
// "test_batchnorm_epsilon_training_mode",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_batchnorm_epsilon",
|
2023-08-12 15:53:10 +00:00
|
|
|
// "test_batchnorm_example_training_mode",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_batchnorm_example",
|
|
|
|
|
// // "test_bernoulli_double_expanded",
|
|
|
|
|
// // "test_bernoulli_double",
|
|
|
|
|
// // "test_bernoulli_expanded",
|
|
|
|
|
// // "test_bernoulli_seed_expanded",
|
|
|
|
|
// // "test_bernoulli_seed",
|
|
|
|
|
// // "test_bernoulli",
|
|
|
|
|
// // "test_bitshift_left_uint16",
|
|
|
|
|
// // "test_bitshift_left_uint32",
|
|
|
|
|
// // "test_bitshift_left_uint64",
|
|
|
|
|
// // "test_bitshift_left_uint8",
|
|
|
|
|
// // "test_bitshift_right_uint16",
|
|
|
|
|
// // "test_bitshift_right_uint32",
|
|
|
|
|
// // "test_bitshift_right_uint64",
|
|
|
|
|
// // "test_bitshift_right_uint8",
|
|
|
|
|
// // "test_blackmanwindow_expanded",
|
|
|
|
|
// // "test_blackmanwindow_symmetric_expanded",
|
|
|
|
|
// // "test_blackmanwindow_symmetric",
|
|
|
|
|
// // "test_blackmanwindow",
|
|
|
|
|
// // "test_cast_BFLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_cast_DOUBLE_to_FLOAT",
|
|
|
|
|
// // "test_cast_DOUBLE_to_FLOAT16",
|
|
|
|
|
// // "test_cast_FLOAT_to_BFLOAT16",
|
|
|
|
|
// // "test_cast_FLOAT_to_DOUBLE",
|
|
|
|
|
// // "test_cast_FLOAT_to_FLOAT16",
|
|
|
|
|
// // "test_cast_FLOAT_to_STRING",
|
|
|
|
|
// // "test_cast_FLOAT16_to_DOUBLE",
|
|
|
|
|
// // "test_cast_FLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_cast_STRING_to_FLOAT",
|
|
|
|
|
// // "test_castlike_BFLOAT16_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_BFLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT16_expanded",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT16",
|
|
|
|
|
// // "test_castlike_FLOAT_to_BFLOAT16_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_BFLOAT16",
|
|
|
|
|
// // "test_castlike_FLOAT_to_DOUBLE_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_DOUBLE",
|
|
|
|
|
// // "test_castlike_FLOAT_to_FLOAT16_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_FLOAT16",
|
|
|
|
|
// // "test_castlike_FLOAT_to_STRING_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_STRING",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_DOUBLE_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_DOUBLE",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_castlike_STRING_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_STRING_to_FLOAT",
|
|
|
|
|
"test_ceil_example",
|
|
|
|
|
"test_ceil",
|
|
|
|
|
// "test_celu_expanded",
|
|
|
|
|
// "test_celu",
|
|
|
|
|
// "test_clip_default_inbounds",
|
|
|
|
|
// "test_clip_default_int8_inbounds",
|
|
|
|
|
// "test_clip_default_int8_max",
|
|
|
|
|
// "test_clip_default_int8_min",
|
|
|
|
|
// "test_clip_default_max",
|
|
|
|
|
// "test_clip_default_min",
|
|
|
|
|
// "test_clip_example",
|
|
|
|
|
// "test_clip_inbounds",
|
|
|
|
|
// "test_clip_outbounds",
|
|
|
|
|
// "test_clip_splitbounds",
|
|
|
|
|
// "test_clip",
|
|
|
|
|
// // "test_compress_0",
|
|
|
|
|
// // "test_compress_1",
|
|
|
|
|
// // "test_compress_default_axis",
|
|
|
|
|
// // "test_compress_negative_axis",
|
2023-09-13 07:05:00 +00:00
|
|
|
"test_concat_1d_axis_0",
|
|
|
|
|
"test_concat_1d_axis_negative_1",
|
|
|
|
|
"test_concat_2d_axis_0",
|
|
|
|
|
"test_concat_2d_axis_1",
|
|
|
|
|
"test_concat_2d_axis_negative_1",
|
|
|
|
|
"test_concat_2d_axis_negative_2",
|
|
|
|
|
"test_concat_3d_axis_0",
|
|
|
|
|
"test_concat_3d_axis_1",
|
|
|
|
|
"test_concat_3d_axis_2",
|
|
|
|
|
"test_concat_3d_axis_negative_1",
|
|
|
|
|
"test_concat_3d_axis_negative_2",
|
|
|
|
|
"test_concat_3d_axis_negative_3",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_conv_with_autopad_same",
|
|
|
|
|
"test_conv_with_strides_and_asymmetric_padding",
|
|
|
|
|
"test_conv_with_strides_no_padding",
|
|
|
|
|
"test_conv_with_strides_padding",
|
|
|
|
|
// // "test_convinteger_with_padding",
|
|
|
|
|
// // "test_convinteger_without_padding",
|
2023-07-08 18:10:50 +00:00
|
|
|
"test_convtranspose_1d",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_convtranspose_3d",
|
2023-07-08 18:10:50 +00:00
|
|
|
"test_convtranspose_autopad_same",
|
|
|
|
|
"test_convtranspose_dilations",
|
|
|
|
|
"test_convtranspose_kernel_shape",
|
|
|
|
|
"opset{9,17}/test_convtranspose_output_shape",
|
|
|
|
|
"test_convtranspose_pad",
|
|
|
|
|
"test_convtranspose_pads",
|
|
|
|
|
"test_convtranspose_with_kernel",
|
|
|
|
|
"test_convtranspose",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_cos_example",
|
|
|
|
|
"test_cos",
|
|
|
|
|
"test_cosh_example",
|
|
|
|
|
"test_cosh",
|
|
|
|
|
// "test_cumsum_1d_exclusive",
|
|
|
|
|
// "test_cumsum_1d_reverse_exclusive",
|
|
|
|
|
// "test_cumsum_1d_reverse",
|
|
|
|
|
// "test_cumsum_1d",
|
|
|
|
|
// "test_cumsum_2d_axis_0",
|
|
|
|
|
// "test_cumsum_2d_axis_1",
|
|
|
|
|
// "test_cumsum_2d_negative_axis",
|
|
|
|
|
// "test_depthtospace_crd_mode_example",
|
|
|
|
|
// "test_depthtospace_crd_mode",
|
|
|
|
|
// "test_depthtospace_dcr_mode",
|
|
|
|
|
// "test_depthtospace_example",
|
|
|
|
|
// "test_depthtospace",
|
|
|
|
|
// // "test_dequantizelinear_axis",
|
|
|
|
|
// // "test_dequantizelinear",
|
|
|
|
|
// // "test_det_2d",
|
|
|
|
|
// // "test_det_nd",
|
|
|
|
|
// // "test_dft_axis",
|
|
|
|
|
// // "test_dft_inverse",
|
|
|
|
|
// // "test_dft",
|
|
|
|
|
"test_div_bcast",
|
|
|
|
|
"test_div_example",
|
|
|
|
|
// "test_div_uint8",
|
|
|
|
|
"test_div",
|
|
|
|
|
// // "test_dropout_default_mask_ratio",
|
|
|
|
|
// // "test_dropout_default_mask",
|
|
|
|
|
// // "test_dropout_default_old",
|
|
|
|
|
// // "test_dropout_default_ratio",
|
|
|
|
|
// // "test_dropout_default",
|
|
|
|
|
// // "test_dropout_random_old",
|
|
|
|
|
// // "test_dropout_random",
|
|
|
|
|
// // "test_dynamic_slice_default_axes",
|
|
|
|
|
// // "test_dynamic_slice_end_out_of_bounds",
|
|
|
|
|
// // "test_dynamic_slice_neg",
|
|
|
|
|
// // "test_dynamic_slice_start_out_of_bounds",
|
|
|
|
|
// // "test_dynamic_slice",
|
|
|
|
|
// // "test_dynamicquantizelinear_expanded",
|
|
|
|
|
// // "test_dynamicquantizelinear_max_adjusted_expanded",
|
|
|
|
|
// // "test_dynamicquantizelinear_max_adjusted",
|
|
|
|
|
// // "test_dynamicquantizelinear_min_adjusted_expanded",
|
|
|
|
|
// // "test_dynamicquantizelinear_min_adjusted",
|
|
|
|
|
// // "test_dynamicquantizelinear",
|
2023-09-14 20:14:11 +00:00
|
|
|
"test_edge_pad",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_einsum_batch_diagonal",
|
|
|
|
|
// "test_einsum_batch_matmul",
|
|
|
|
|
// "test_einsum_inner_prod",
|
|
|
|
|
// "test_einsum_sum",
|
|
|
|
|
// "test_einsum_transpose",
|
|
|
|
|
"test_elu_default",
|
|
|
|
|
"test_elu_example",
|
|
|
|
|
"test_elu",
|
|
|
|
|
// "test_equal_bcast",
|
|
|
|
|
// "test_equal",
|
|
|
|
|
"test_erf",
|
2023-04-27 22:04:09 +00:00
|
|
|
"test_exp_example",
|
|
|
|
|
"test_exp",
|
2023-07-11 16:38:16 +00:00
|
|
|
"test_expand_dim_changed",
|
|
|
|
|
"test_expand_dim_unchanged",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_eyelike_populate_off_main_diagonal",
|
|
|
|
|
// "test_eyelike_with_dtype",
|
|
|
|
|
// "test_eyelike_without_dtype",
|
2023-07-27 19:50:45 +00:00
|
|
|
"test_flatten_axis0",
|
|
|
|
|
"test_flatten_axis1",
|
|
|
|
|
"test_flatten_axis2",
|
|
|
|
|
"test_flatten_axis3",
|
|
|
|
|
"test_flatten_default_axis",
|
|
|
|
|
"test_flatten_negative_axis1",
|
|
|
|
|
"test_flatten_negative_axis2",
|
|
|
|
|
"test_flatten_negative_axis3",
|
|
|
|
|
"test_flatten_negative_axis4",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_floor_example",
|
|
|
|
|
"test_floor",
|
2023-08-03 21:09:37 +00:00
|
|
|
"test_gather_0",
|
|
|
|
|
"test_gather_1",
|
|
|
|
|
"test_gather_2d_indices",
|
|
|
|
|
"test_gather_negative_indices",
|
2023-08-28 16:55:25 +00:00
|
|
|
"test_gather_elements_0",
|
|
|
|
|
"test_gather_elements_1",
|
|
|
|
|
"test_gather_elements_negative_indices",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_gather_negative_indices",
|
|
|
|
|
// // "test_gathernd_example_float32",
|
|
|
|
|
// // "test_gathernd_example_int32_batch_dim1",
|
|
|
|
|
// // "test_gathernd_example_int32",
|
|
|
|
|
"test_gemm_all_attributes",
|
|
|
|
|
"test_gemm_alpha",
|
|
|
|
|
"test_gemm_beta",
|
|
|
|
|
"test_gemm_broadcast",
|
|
|
|
|
"test_gemm_default_matrix_bias",
|
|
|
|
|
"test_gemm_default_no_bias",
|
|
|
|
|
"test_gemm_default_scalar_bias",
|
|
|
|
|
"test_gemm_default_single_elem_vector_bias",
|
|
|
|
|
"test_gemm_default_vector_bias",
|
|
|
|
|
"test_gemm_default_zero_bias",
|
|
|
|
|
"test_gemm_nobroadcast",
|
|
|
|
|
"test_gemm_transposeA",
|
|
|
|
|
"test_gemm_transposeB",
|
|
|
|
|
"test_globalaveragepool_precomputed",
|
|
|
|
|
"test_globalaveragepool",
|
|
|
|
|
"test_globalmaxpool_precomputed",
|
|
|
|
|
"test_globalmaxpool",
|
2023-08-25 19:11:25 +00:00
|
|
|
"test_greater_bcast",
|
|
|
|
|
"test_greater_equal_bcast_expanded",
|
|
|
|
|
"test_greater_equal_bcast",
|
|
|
|
|
"test_greater_equal_expanded",
|
|
|
|
|
"test_greater_equal",
|
|
|
|
|
"test_greater",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_gridsample_aligncorners_true",
|
|
|
|
|
// // "test_gridsample_bicubic",
|
|
|
|
|
// // "test_gridsample_bilinear",
|
|
|
|
|
// // "test_gridsample_border_padding",
|
|
|
|
|
// // "test_gridsample_nearest",
|
|
|
|
|
// // "test_gridsample_reflection_padding",
|
|
|
|
|
// // "test_gridsample_zeros_padding",
|
|
|
|
|
// // "test_gridsample",
|
|
|
|
|
// // "test_gru_batchwise",
|
|
|
|
|
// // "test_gru_defaults",
|
|
|
|
|
// // "test_gru_seq_length",
|
|
|
|
|
// // "test_gru_with_initial_bias",
|
|
|
|
|
// // "test_hammingwindow_expanded",
|
|
|
|
|
// // "test_hammingwindow_symmetric_expanded",
|
|
|
|
|
// // "test_hammingwindow_symmetric",
|
|
|
|
|
// // "test_hammingwindow",
|
|
|
|
|
// // "test_hannwindow_expanded",
|
|
|
|
|
// // "test_hannwindow_symmetric_expanded",
|
|
|
|
|
// // "test_hannwindow_symmetric",
|
|
|
|
|
// // "test_hannwindow",
|
|
|
|
|
// // "test_hardmax_axis_0",
|
|
|
|
|
// // "test_hardmax_axis_1",
|
|
|
|
|
// // "test_hardmax_axis_2",
|
|
|
|
|
// // "test_hardmax_default_axis",
|
|
|
|
|
// // "test_hardmax_example",
|
|
|
|
|
// // "test_hardmax_negative_axis",
|
|
|
|
|
// // "test_hardmax_one_hot",
|
|
|
|
|
// // "test_hardsigmoid_default",
|
|
|
|
|
// // "test_hardsigmoid_example",
|
|
|
|
|
// // "test_hardsigmoid",
|
|
|
|
|
// // "test_hardswish_expanded",
|
|
|
|
|
// // "test_hardswish",
|
2023-09-19 19:20:18 +00:00
|
|
|
"test_if",
|
|
|
|
|
// TODO: Uncomment 'test_if_seq' and 'test_if_opt' once the test infra
|
|
|
|
|
// supports Sequence and Optional types
|
|
|
|
|
// "test_if_seq",
|
|
|
|
|
// "test_if_opt",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_instancenorm_epsilon",
|
|
|
|
|
"test_instancenorm_example",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_isinf_negative",
|
|
|
|
|
// "test_isinf_positive",
|
|
|
|
|
// "test_isinf",
|
|
|
|
|
// "test_isnan",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_2d_axis_negative_1_expanded",
|
|
|
|
|
"test_layer_normalization_2d_axis_negative_1",
|
|
|
|
|
"test_layer_normalization_2d_axis_negative_2_expanded",
|
|
|
|
|
"test_layer_normalization_2d_axis_negative_2",
|
|
|
|
|
"test_layer_normalization_2d_axis0_expanded",
|
|
|
|
|
"test_layer_normalization_2d_axis0",
|
|
|
|
|
"test_layer_normalization_2d_axis1_expanded",
|
|
|
|
|
"test_layer_normalization_2d_axis1",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_3d_axis_negative_1_epsilon_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_3d_axis_negative_1_epsilon",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_3d_axis_negative_2_epsilon_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_3d_axis_negative_2_epsilon",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_3d_axis_negative_3_epsilon_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_3d_axis_negative_3_epsilon",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_3d_axis0_epsilon_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_3d_axis0_epsilon",
|
|
|
|
|
"test_layer_normalization_3d_axis1_epsilon_expanded",
|
|
|
|
|
"test_layer_normalization_3d_axis1_epsilon",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_3d_axis2_epsilon_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_3d_axis2_epsilon",
|
|
|
|
|
"test_layer_normalization_4d_axis_negative_1_expanded",
|
|
|
|
|
"test_layer_normalization_4d_axis_negative_1",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_4d_axis_negative_2_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_4d_axis_negative_2",
|
|
|
|
|
"test_layer_normalization_4d_axis_negative_3_expanded",
|
|
|
|
|
"test_layer_normalization_4d_axis_negative_3",
|
|
|
|
|
"test_layer_normalization_4d_axis_negative_4_expanded",
|
|
|
|
|
"test_layer_normalization_4d_axis_negative_4",
|
|
|
|
|
"test_layer_normalization_4d_axis0_expanded",
|
|
|
|
|
"test_layer_normalization_4d_axis0",
|
|
|
|
|
"test_layer_normalization_4d_axis1_expanded",
|
|
|
|
|
"test_layer_normalization_4d_axis1",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_layer_normalization_4d_axis2_expanded",
|
2023-08-08 16:09:37 +00:00
|
|
|
"test_layer_normalization_4d_axis2",
|
|
|
|
|
"test_layer_normalization_4d_axis3_expanded",
|
|
|
|
|
"test_layer_normalization_4d_axis3",
|
|
|
|
|
"test_layer_normalization_default_axis_expanded",
|
|
|
|
|
"test_layer_normalization_default_axis",
|
2023-04-26 22:11:01 +00:00
|
|
|
"test_leakyrelu_default",
|
|
|
|
|
"test_leakyrelu_example",
|
|
|
|
|
"test_leakyrelu",
|
2023-08-25 19:11:25 +00:00
|
|
|
"test_less_bcast",
|
|
|
|
|
"test_less_equal_bcast_expanded",
|
|
|
|
|
"test_less_equal_bcast",
|
|
|
|
|
"test_less_equal_expanded",
|
|
|
|
|
"test_less_equal",
|
|
|
|
|
"test_less",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_log_example",
|
|
|
|
|
"test_log",
|
|
|
|
|
// // "test_logsoftmax_axis_0_expanded",
|
|
|
|
|
// // "test_logsoftmax_axis_0",
|
|
|
|
|
// // "test_logsoftmax_axis_1_expanded",
|
|
|
|
|
// // "test_logsoftmax_axis_1",
|
|
|
|
|
// // "test_logsoftmax_axis_2_expanded",
|
|
|
|
|
// // "test_logsoftmax_axis_2",
|
|
|
|
|
// // "test_logsoftmax_default_axis_expanded",
|
|
|
|
|
// // "test_logsoftmax_default_axis",
|
|
|
|
|
// // "test_logsoftmax_example_1_expanded",
|
|
|
|
|
// // "test_logsoftmax_example_1",
|
|
|
|
|
// // "test_logsoftmax_large_number_expanded",
|
|
|
|
|
// // "test_logsoftmax_large_number",
|
|
|
|
|
// // "test_logsoftmax_negative_axis_expanded",
|
|
|
|
|
// // "test_logsoftmax_negative_axis",
|
|
|
|
|
// "test_lrn_default",
|
|
|
|
|
// "test_lrn",
|
|
|
|
|
// // "test_lstm_batchwise",
|
|
|
|
|
// // "test_lstm_defaults",
|
|
|
|
|
// // "test_lstm_with_initial_bias",
|
|
|
|
|
// // "test_lstm_with_peepholes",
|
|
|
|
|
"test_matmul_2d",
|
|
|
|
|
"test_matmul_3d",
|
|
|
|
|
"test_matmul_4d",
|
|
|
|
|
// // "test_matmulinteger",
|
|
|
|
|
// "test_max_example",
|
|
|
|
|
// "test_max_float16",
|
|
|
|
|
// "test_max_float32",
|
|
|
|
|
// "test_max_float64",
|
|
|
|
|
// "test_max_int16",
|
|
|
|
|
// "test_max_int32",
|
|
|
|
|
// "test_max_int64",
|
|
|
|
|
// "test_max_int8",
|
|
|
|
|
// "test_max_one_input",
|
|
|
|
|
// "test_max_two_inputs",
|
|
|
|
|
// "test_max_uint16",
|
|
|
|
|
// "test_max_uint32",
|
|
|
|
|
// "test_max_uint64",
|
|
|
|
|
// "test_max_uint8",
|
|
|
|
|
// "test_maxpool_1d_default",
|
|
|
|
|
// "test_maxpool_2d_ceil",
|
|
|
|
|
"test_maxpool_2d_default",
|
|
|
|
|
// "test_maxpool_2d_dilations",
|
|
|
|
|
"test_maxpool_2d_pads",
|
|
|
|
|
"test_maxpool_2d_precomputed_pads",
|
|
|
|
|
"test_maxpool_2d_precomputed_same_upper",
|
|
|
|
|
"test_maxpool_2d_precomputed_strides",
|
|
|
|
|
"test_maxpool_2d_same_lower",
|
|
|
|
|
"test_maxpool_2d_same_upper",
|
|
|
|
|
"test_maxpool_2d_strides",
|
|
|
|
|
// "test_maxpool_2d_uint8",
|
|
|
|
|
// "test_maxpool_3d_default",
|
|
|
|
|
// "test_maxpool_with_argmax_2d_precomputed_pads",
|
|
|
|
|
// "test_maxpool_with_argmax_2d_precomputed_strides",
|
|
|
|
|
// // "test_maxunpool_export_with_output_shape",
|
|
|
|
|
// // "test_maxunpool_export_without_output_shape",
|
|
|
|
|
// // "test_mean_example",
|
|
|
|
|
// // "test_mean_one_input",
|
|
|
|
|
// // "test_mean_two_inputs",
|
|
|
|
|
// // "test_melweightmatrix",
|
|
|
|
|
// "test_min_example",
|
|
|
|
|
// "test_min_float16",
|
|
|
|
|
// "test_min_float32",
|
|
|
|
|
// "test_min_float64",
|
|
|
|
|
// "test_min_int16",
|
|
|
|
|
// "test_min_int32",
|
|
|
|
|
// "test_min_int64",
|
|
|
|
|
// "test_min_int8",
|
|
|
|
|
// "test_min_one_input",
|
|
|
|
|
// "test_min_two_inputs",
|
|
|
|
|
// "test_min_uint16",
|
|
|
|
|
// "test_min_uint32",
|
|
|
|
|
// "test_min_uint64",
|
|
|
|
|
// "test_min_uint8",
|
|
|
|
|
// "test_mod_bcast",
|
|
|
|
|
// "test_mod_broadcast",
|
|
|
|
|
// "test_mod_float_mixed_sign_example",
|
|
|
|
|
// "test_mod_fmod_mixed_sign_example",
|
|
|
|
|
// "test_mod_int64_fmod",
|
|
|
|
|
// "test_mod_int64_mixed_sign_example",
|
|
|
|
|
// "test_mod_mixed_sign_float16",
|
|
|
|
|
// "test_mod_mixed_sign_float32",
|
|
|
|
|
// "test_mod_mixed_sign_float64",
|
|
|
|
|
// "test_mod_mixed_sign_int16",
|
|
|
|
|
// "test_mod_mixed_sign_int32",
|
|
|
|
|
// "test_mod_mixed_sign_int64",
|
|
|
|
|
// "test_mod_mixed_sign_int8",
|
|
|
|
|
// "test_mod_uint16",
|
|
|
|
|
// "test_mod_uint32",
|
|
|
|
|
// "test_mod_uint64",
|
|
|
|
|
// "test_mod_uint8",
|
|
|
|
|
// // "test_momentum_multiple",
|
|
|
|
|
// // "test_momentum",
|
|
|
|
|
"test_mul_bcast",
|
|
|
|
|
"test_mul_example",
|
|
|
|
|
// "test_mul_uint8",
|
|
|
|
|
"test_mul",
|
|
|
|
|
// "test_mvn_expanded",
|
|
|
|
|
// "test_mvn",
|
|
|
|
|
"test_neg_example",
|
|
|
|
|
"test_neg",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NC_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NC",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_weight",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// // "test_nesterov_momentum",
|
|
|
|
|
// // "test_nllloss_NC_expanded",
|
|
|
|
|
// // "test_nllloss_NC",
|
|
|
|
|
// // "test_nllloss_NCd1_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_ii",
|
|
|
|
|
// // "test_nllloss_NCd1_mean_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_mean_weight_negative_ii",
|
|
|
|
|
// // "test_nllloss_NCd1_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_weight_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_weight_ii",
|
|
|
|
|
// // "test_nllloss_NCd1_weight",
|
|
|
|
|
// // "test_nllloss_NCd1",
|
|
|
|
|
// // "test_nllloss_NCd1d2_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_no_weight_reduction_mean_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_mean_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_mean",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_sum_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_sum",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_mean",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight",
|
|
|
|
|
// // "test_nllloss_NCd1d2",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_none_no_weight_negative_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_sum_weight_high_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// "test_nonmaxsuppression_center_point_box_format",
|
|
|
|
|
// "test_nonmaxsuppression_flipped_coordinates",
|
|
|
|
|
// "test_nonmaxsuppression_identical_boxes",
|
|
|
|
|
// "test_nonmaxsuppression_limit_output_size",
|
|
|
|
|
// "test_nonmaxsuppression_single_box",
|
|
|
|
|
// "test_nonmaxsuppression_suppress_by_IOU_and_scores",
|
|
|
|
|
// "test_nonmaxsuppression_suppress_by_IOU",
|
|
|
|
|
// "test_nonmaxsuppression_two_batches",
|
|
|
|
|
// "test_nonmaxsuppression_two_classes",
|
|
|
|
|
// "test_nonzero_example",
|
2023-08-28 02:50:17 +00:00
|
|
|
"test_not_2d",
|
|
|
|
|
"test_not_3d",
|
|
|
|
|
"test_not_4d",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_onehot_negative_indices",
|
|
|
|
|
// // "test_onehot_with_axis",
|
|
|
|
|
// // "test_onehot_with_negative_axis",
|
|
|
|
|
// // "test_onehot_without_axis",
|
|
|
|
|
// // "test_optional_get_element_sequence",
|
|
|
|
|
// // "test_optional_get_element",
|
|
|
|
|
// // "test_optional_has_element_empty",
|
|
|
|
|
// // "test_optional_has_element",
|
|
|
|
|
// "test_or_bcast3v1d",
|
|
|
|
|
// "test_or_bcast3v2d",
|
|
|
|
|
// "test_or_bcast4v2d",
|
|
|
|
|
// "test_or_bcast4v3d",
|
|
|
|
|
// "test_or_bcast4v4d",
|
|
|
|
|
// "test_or2d",
|
|
|
|
|
// "test_or3d",
|
|
|
|
|
// "test_or4d",
|
|
|
|
|
"test_pow_bcast_array",
|
|
|
|
|
"test_pow_bcast_scalar",
|
|
|
|
|
"test_pow_example",
|
|
|
|
|
// "test_pow_types_float",
|
|
|
|
|
// "test_pow_types_float32_int32",
|
|
|
|
|
// "test_pow_types_float32_int64",
|
|
|
|
|
// "test_pow_types_float32_uint32",
|
|
|
|
|
// "test_pow_types_float32_uint64",
|
|
|
|
|
// "test_pow_types_int",
|
|
|
|
|
// "test_pow_types_int32_float32",
|
|
|
|
|
// "test_pow_types_int32_int32",
|
|
|
|
|
// "test_pow_types_int64_float32",
|
|
|
|
|
// "test_pow_types_int64_int64",
|
|
|
|
|
"test_pow",
|
|
|
|
|
// "test_prelu_broadcast",
|
|
|
|
|
// "test_prelu_example",
|
|
|
|
|
// // "test_qlinearconv",
|
|
|
|
|
// // "test_qlinearmatmul_2D",
|
|
|
|
|
// // "test_qlinearmatmul_3D",
|
|
|
|
|
// // "test_quantizelinear_axis",
|
|
|
|
|
// // "test_quantizelinear",
|
2023-09-30 09:05:32 +00:00
|
|
|
"test_range_float_type_positive_delta_expanded",
|
|
|
|
|
"test_range_float_type_positive_delta",
|
|
|
|
|
"test_range_int32_type_negative_delta_expanded",
|
|
|
|
|
"test_range_int32_type_negative_delta",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_reciprocal_example",
|
|
|
|
|
"test_reciprocal",
|
2023-07-07 16:55:55 +00:00
|
|
|
"test_reduce_l1_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_l1_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_l1_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_l1_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_l1_keep_dims_example",
|
|
|
|
|
"test_reduce_l1_keep_dims_random",
|
|
|
|
|
"test_reduce_l1_negative_axes_keep_dims_example",
|
|
|
|
|
"test_reduce_l1_negative_axes_keep_dims_random",
|
|
|
|
|
"test_reduce_l2_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_l2_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_l2_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_l2_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_l2_keep_dims_example",
|
|
|
|
|
"test_reduce_l2_keep_dims_random",
|
|
|
|
|
"test_reduce_l2_negative_axes_keep_dims_example",
|
|
|
|
|
"test_reduce_l2_negative_axes_keep_dims_random",
|
2023-06-12 14:46:27 +00:00
|
|
|
"test_reduce_log_sum_asc_axes",
|
|
|
|
|
"test_reduce_log_sum_default",
|
|
|
|
|
"test_reduce_log_sum_desc_axes",
|
2023-09-09 00:03:04 +00:00
|
|
|
// tests "test_reduce_log_sum_exp_*" on opset17/opset18 are excluded because they use float64.
|
|
|
|
|
"opset{7,8,9}/test_reduce_log_sum_exp_default_axes_keepdims_example",
|
|
|
|
|
"opset{7,8,9}/test_reduce_log_sum_exp_default_axes_keepdims_random",
|
|
|
|
|
"opset{7,8,9}/test_reduce_log_sum_exp_do_not_keepdims_example",
|
|
|
|
|
"opset{7,8,9}/test_reduce_log_sum_exp_do_not_keepdims_random",
|
|
|
|
|
"opset{7,8,9}/test_reduce_log_sum_exp_keepdims_example",
|
|
|
|
|
"opset{7,8,9}/test_reduce_log_sum_exp_keepdims_random",
|
|
|
|
|
"opset11/test_reduce_log_sum_exp_negative_axes_keepdims_example",
|
|
|
|
|
"opset11/test_reduce_log_sum_exp_negative_axes_keepdims_random",
|
2023-06-12 14:46:27 +00:00
|
|
|
"test_reduce_log_sum_negative_axes",
|
|
|
|
|
"test_reduce_log_sum",
|
|
|
|
|
"test_reduce_max_default_axes_keepdim_example",
|
|
|
|
|
"test_reduce_max_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_max_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_max_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_max_keepdims_example",
|
|
|
|
|
"test_reduce_max_keepdims_random",
|
|
|
|
|
"test_reduce_max_negative_axes_keepdims_example",
|
|
|
|
|
"test_reduce_max_negative_axes_keepdims_random",
|
|
|
|
|
"test_reduce_mean_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_mean_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_mean_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_mean_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_mean_keepdims_example",
|
|
|
|
|
"test_reduce_mean_keepdims_random",
|
|
|
|
|
"test_reduce_mean_negative_axes_keepdims_example",
|
|
|
|
|
"test_reduce_mean_negative_axes_keepdims_random",
|
|
|
|
|
"test_reduce_min_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_min_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_min_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_min_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_min_keepdims_example",
|
|
|
|
|
"test_reduce_min_keepdims_random",
|
|
|
|
|
"test_reduce_min_negative_axes_keepdims_example",
|
|
|
|
|
"test_reduce_min_negative_axes_keepdims_random",
|
|
|
|
|
"test_reduce_prod_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_prod_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_prod_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_prod_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_prod_keepdims_example",
|
|
|
|
|
"test_reduce_prod_keepdims_random",
|
|
|
|
|
"test_reduce_prod_negative_axes_keepdims_example",
|
|
|
|
|
"test_reduce_prod_negative_axes_keepdims_random",
|
|
|
|
|
"test_reduce_sum_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_sum_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_sum_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_sum_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_sum_empty_axes_input_noop_example",
|
|
|
|
|
"test_reduce_sum_empty_axes_input_noop_random",
|
|
|
|
|
"test_reduce_sum_keepdims_example",
|
|
|
|
|
"test_reduce_sum_keepdims_random",
|
|
|
|
|
"test_reduce_sum_negative_axes_keepdims_example",
|
|
|
|
|
"test_reduce_sum_negative_axes_keepdims_random",
|
|
|
|
|
"test_reduce_sum_square_default_axes_keepdims_example",
|
|
|
|
|
"test_reduce_sum_square_default_axes_keepdims_random",
|
|
|
|
|
"test_reduce_sum_square_do_not_keepdims_example",
|
|
|
|
|
"test_reduce_sum_square_do_not_keepdims_random",
|
|
|
|
|
"test_reduce_sum_square_keepdims_example",
|
|
|
|
|
"test_reduce_sum_square_keepdims_random",
|
|
|
|
|
"test_reduce_sum_square_negative_axes_keepdims_example",
|
|
|
|
|
"test_reduce_sum_square_negative_axes_keepdims_random",
|
2023-09-14 20:14:11 +00:00
|
|
|
"test_reflect_pad",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
"test_relu",
|
|
|
|
|
// "test_reshape_allowzero_reordered",
|
|
|
|
|
"test_reshape_extended_dims",
|
|
|
|
|
"test_reshape_negative_dim",
|
|
|
|
|
"test_reshape_negative_extended_dims",
|
|
|
|
|
"test_reshape_one_dim",
|
|
|
|
|
"test_reshape_reduced_dims",
|
|
|
|
|
"test_reshape_reordered_all_dims",
|
|
|
|
|
"test_reshape_reordered_dims",
|
|
|
|
|
"test_reshape_reordered_last_dims",
|
|
|
|
|
"test_reshape_zero_and_negative_dim",
|
|
|
|
|
"test_reshape_zero_dim",
|
2023-07-31 16:35:06 +00:00
|
|
|
"test_resize_downsample_linear",
|
|
|
|
|
"test_resize_downsample_nearest",
|
2023-08-10 16:14:43 +00:00
|
|
|
"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_resize_downsample_scales_cubic_align_corners",
|
2023-08-10 16:14:43 +00:00
|
|
|
"test_resize_downsample_scales_cubic",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_resize_downsample_scales_linear_align_corners",
|
2023-08-10 16:14:43 +00:00
|
|
|
"test_resize_downsample_scales_linear",
|
|
|
|
|
"test_resize_downsample_scales_nearest",
|
|
|
|
|
"test_resize_downsample_sizes_cubic",
|
|
|
|
|
"test_resize_downsample_sizes_linear_pytorch_half_pixel",
|
|
|
|
|
"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn",
|
|
|
|
|
"test_resize_downsample_sizes_nearest",
|
2023-07-31 16:35:06 +00:00
|
|
|
"test_resize_nearest",
|
2023-08-10 16:14:43 +00:00
|
|
|
"test_resize_tf_crop_and_resize",
|
2023-07-31 16:35:06 +00:00
|
|
|
"test_resize_upsample_linear",
|
|
|
|
|
"test_resize_upsample_nearest",
|
2023-08-10 16:14:43 +00:00
|
|
|
"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside",
|
|
|
|
|
"test_resize_upsample_scales_cubic_align_corners",
|
|
|
|
|
"test_resize_upsample_scales_cubic_asymmetric",
|
|
|
|
|
"test_resize_upsample_scales_cubic",
|
|
|
|
|
"test_resize_upsample_scales_linear_align_corners",
|
|
|
|
|
"test_resize_upsample_scales_linear",
|
|
|
|
|
"test_resize_upsample_scales_nearest",
|
|
|
|
|
"test_resize_upsample_sizes_cubic",
|
|
|
|
|
"opset{12,13,17,18}/test_resize_upsample_sizes_nearest_ceil_half_pixel",
|
|
|
|
|
"opset{12,13,17,18}/test_resize_upsample_sizes_nearest_floor_align_corners",
|
|
|
|
|
"opset{12,13,17,18}/test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric",
|
|
|
|
|
"test_resize_upsample_sizes_nearest",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_reversesequence_batch",
|
|
|
|
|
// // "test_reversesequence_time",
|
|
|
|
|
// // "test_rnn_seq_length",
|
|
|
|
|
// // "test_roialign_aligned_false",
|
|
|
|
|
// // "test_roialign_aligned_true",
|
|
|
|
|
// // "test_roialign",
|
|
|
|
|
// // "test_round",
|
|
|
|
|
// // "test_scan_sum",
|
|
|
|
|
// // "test_scan9_sum",
|
|
|
|
|
// // "test_scatter_elements_with_axis",
|
|
|
|
|
// // "test_scatter_elements_with_duplicate_indices",
|
|
|
|
|
// // "test_scatter_elements_with_negative_indices",
|
|
|
|
|
// // "test_scatter_elements_without_axis",
|
|
|
|
|
// // "test_scatter_with_axis",
|
|
|
|
|
// // "test_scatter_without_axis",
|
|
|
|
|
// // "test_scatternd_add",
|
|
|
|
|
// // "test_scatternd_multiply",
|
|
|
|
|
// // "test_scatternd",
|
|
|
|
|
// // "test_sce_mean_3d_expanded",
|
|
|
|
|
// // "test_sce_mean_3d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_3d_log_prob",
|
|
|
|
|
// // "test_sce_mean_3d",
|
|
|
|
|
// // "test_sce_mean_expanded",
|
|
|
|
|
// // "test_sce_mean_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii",
|
|
|
|
|
// // "test_sce_mean_weight_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d",
|
|
|
|
|
// // "test_sce_mean_weight_ii_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight_ii",
|
|
|
|
|
// // "test_sce_mean_weight_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight",
|
|
|
|
|
// // "test_sce_mean",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii_log_prob",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// // "test_sce_none_expanded",
|
|
|
|
|
// // "test_sce_none_log_prob_expanded",
|
|
|
|
|
// // "test_sce_none_log_prob",
|
|
|
|
|
// // "test_sce_none_weights_expanded",
|
|
|
|
|
// // "test_sce_none_weights_log_prob_expanded",
|
|
|
|
|
// // "test_sce_none_weights_log_prob",
|
|
|
|
|
// // "test_sce_none_weights",
|
|
|
|
|
// // "test_sce_none",
|
|
|
|
|
// // "test_sce_sum_expanded",
|
|
|
|
|
// // "test_sce_sum_log_prob_expanded",
|
|
|
|
|
// // "test_sce_sum_log_prob",
|
|
|
|
|
// // "test_sce_sum",
|
|
|
|
|
// "test_selu_default",
|
|
|
|
|
// "test_selu_example",
|
|
|
|
|
// "test_selu",
|
|
|
|
|
// // "test_sequence_insert_at_back",
|
|
|
|
|
// // "test_sequence_insert_at_front",
|
|
|
|
|
// // "test_sequence_map_add_1_sequence_1_tensor_expanded",
|
|
|
|
|
// // "test_sequence_map_add_1_sequence_1_tensor",
|
|
|
|
|
// // "test_sequence_map_add_2_sequences_expanded",
|
|
|
|
|
// // "test_sequence_map_add_2_sequences",
|
|
|
|
|
// // "test_sequence_map_extract_shapes_expanded",
|
|
|
|
|
// // "test_sequence_map_extract_shapes",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence_1_tensor_expanded",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence_1_tensor",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence_expanded",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence",
|
|
|
|
|
// // "test_sequence_map_identity_2_sequences_expanded",
|
|
|
|
|
// // "test_sequence_map_identity_2_sequences",
|
|
|
|
|
// "test_shrink_hard",
|
|
|
|
|
// "test_shrink_soft",
|
|
|
|
|
"test_sigmoid_example",
|
|
|
|
|
"test_sigmoid",
|
|
|
|
|
// "test_sign",
|
|
|
|
|
// "test_simple_rnn_batchwise",
|
|
|
|
|
// "test_simple_rnn_defaults",
|
|
|
|
|
// "test_simple_rnn_with_initial_bias",
|
|
|
|
|
"test_sin_example",
|
|
|
|
|
"test_sin",
|
|
|
|
|
"test_sinh_example",
|
|
|
|
|
"test_sinh",
|
|
|
|
|
// // "test_size_example",
|
|
|
|
|
// // "test_size",
|
2023-07-25 21:19:20 +00:00
|
|
|
"test_slice_default_axes",
|
|
|
|
|
"test_slice_default_steps",
|
2023-08-03 22:48:47 +00:00
|
|
|
// "test_slice_end_out_of_bounds",
|
2023-07-25 21:19:20 +00:00
|
|
|
"test_slice_neg_steps",
|
|
|
|
|
"test_slice_neg",
|
|
|
|
|
"test_slice_negative_axes",
|
2023-08-03 22:48:47 +00:00
|
|
|
// "test_slice_start_out_of_bounds",
|
2023-07-25 21:19:20 +00:00
|
|
|
"test_slice",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_softmax_axis_0_expanded",
|
|
|
|
|
// "test_softmax_axis_0",
|
|
|
|
|
// "test_softmax_axis_1_expanded",
|
|
|
|
|
// "test_softmax_axis_1",
|
2023-08-03 01:16:19 +00:00
|
|
|
"test_softmax_axis_2_expanded",
|
|
|
|
|
"test_softmax_axis_2",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight",
|
|
|
|
|
// "test_softmax_cross_entropy_mean",
|
|
|
|
|
// "test_softmax_cross_entropy_none_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights",
|
|
|
|
|
// "test_softmax_cross_entropy_none",
|
|
|
|
|
// "test_softmax_cross_entropy_sum_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_sum_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_sum_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_sum",
|
2023-08-03 01:16:19 +00:00
|
|
|
"opset13/test_softmax_default_axis_expanded",
|
|
|
|
|
"opset13/test_softmax_default_axis",
|
|
|
|
|
"test_softmax_example_expanded",
|
|
|
|
|
"test_softmax_example",
|
|
|
|
|
"test_softmax_large_number_expanded",
|
|
|
|
|
"test_softmax_large_number",
|
|
|
|
|
"test_softmax_negative_axis_expanded",
|
|
|
|
|
"test_softmax_negative_axis",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_softplus_example",
|
|
|
|
|
// // "test_softplus",
|
|
|
|
|
// // "test_softsign_example",
|
|
|
|
|
// // "test_softsign",
|
|
|
|
|
// "test_spacetodepth_example",
|
|
|
|
|
// "test_spacetodepth",
|
2023-07-07 19:16:10 +00:00
|
|
|
"test_split_equal_parts_1d",
|
|
|
|
|
"test_split_equal_parts_2d",
|
|
|
|
|
"test_split_equal_parts_default_axis",
|
|
|
|
|
"test_split_variable_parts_1d",
|
|
|
|
|
"test_split_variable_parts_2d",
|
|
|
|
|
"test_split_variable_parts_default_axis",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_split_zero_size_splits",
|
|
|
|
|
"test_sqrt_example",
|
|
|
|
|
"test_sqrt",
|
[JS/WebGPU] Unsqueeze operator implementation (#16138)
### Description
This PR adds an implementation of the Squeeze operator to WebGPU JSEP.
The implementation follows the [operator
schema](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Unsqueeze).
To implement the `Unsqueeze` operator in the same fashion as the
`Squeeze`, I added the `ComputeOutputShape()` method to the
`UnsqueezeBase` class and made some slight modifications. Please let me
know if it is a bad idea and if I should move this method to the JS
implementation.
I also uncommented test case lines in the `suite-test-list.jsonc` file
for both Squeeze and Unsqueeze operators following @hariharans29's
[comment](https://github.com/microsoft/onnxruntime/pull/16024#issuecomment-1565113633).
### How was it tested
1. I created a model with only one operator:
```Python
import onnx.helper
node = onnx.helper.make_node(
"Unsqueeze",
inputs=["T", "axes"],
outputs=["y"],
)
graph = onnx.helper.make_graph([node], "test", [onnx.helper.make_tensor_value_info("T", 1, [3, 4, 5]), onnx.helper.make_tensor_value_info("axes", 7, [2])], [onnx.helper.make_tensor_value_info("y", 1, [3, 1, 4, 5, 1])])
onnx.save(onnx.helper.make_model(graph), "unsqueeze.onnx")
```
2. I compiled the runtime using @fs-eire's
[instructions](https://gist.github.com/fs-eire/a55b2c7e10a6864b9602c279b8b75dce).
3. I ran the test models in the browser using this minimal setup:
```HTML
<html>
<script src=".\dist\ort.webgpu.min.js"></script>
<script>
async function run() {
const session = await ort.InferenceSession.create('unsqueeze.onnx', {executionProviders: ['webgpu']});
console.log(session);
const input = new ort.Tensor('float32', new Float32Array(60), [3, 4, 5]);
const dim = new ort.Tensor('int64', [1n, 4n], [2]);
const output = await session.run({ "T": input, "axes": dim });
console.log(output);
}
run();
</script>
</html>
```
### Motivation and Context
Improve operator coverage for WebGPU JSEP.
2023-06-01 19:23:02 +00:00
|
|
|
"test_squeeze_negative_axes",
|
|
|
|
|
"test_squeeze",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_stft_with_window",
|
|
|
|
|
// // "test_stft",
|
|
|
|
|
// // "test_strnormalizer_export_monday_casesensintive_lower",
|
|
|
|
|
// // "test_strnormalizer_export_monday_casesensintive_nochangecase",
|
|
|
|
|
// // "test_strnormalizer_export_monday_casesensintive_upper",
|
|
|
|
|
// // "test_strnormalizer_export_monday_empty_output",
|
|
|
|
|
// // "test_strnormalizer_export_monday_insensintive_upper_twodim",
|
|
|
|
|
// // "test_strnormalizer_nostopwords_nochangecase",
|
|
|
|
|
"test_sub_bcast",
|
|
|
|
|
"test_sub_example",
|
|
|
|
|
// "test_sub_uint8",
|
|
|
|
|
"test_sub",
|
|
|
|
|
// "test_sum_example",
|
|
|
|
|
// "test_sum_one_input",
|
|
|
|
|
// "test_sum_two_inputs",
|
|
|
|
|
"test_tan_example",
|
|
|
|
|
"test_tan",
|
|
|
|
|
"test_tanh_example",
|
|
|
|
|
"test_tanh",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_batch_onlybigrams_skip0",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_batch_onlybigrams_skip5",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_batch_uniandbigrams_skip5",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_only_bigrams_skip0",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_onlybigrams_levelempty",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_onlybigrams_skip5",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_uniandbigrams_skip5",
|
2023-04-26 22:11:01 +00:00
|
|
|
"test_thresholdedrelu_default",
|
|
|
|
|
"test_thresholdedrelu_example",
|
|
|
|
|
"test_thresholdedrelu",
|
2023-08-18 17:07:21 +00:00
|
|
|
"test_tile_precomputed",
|
|
|
|
|
"test_tile",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// // "test_top_k_negative_axis",
|
|
|
|
|
// // "test_top_k_smallest",
|
|
|
|
|
// // "test_top_k",
|
|
|
|
|
// // "test_training_dropout_default_mask",
|
|
|
|
|
// // "test_training_dropout_default",
|
|
|
|
|
// // "test_training_dropout_mask",
|
|
|
|
|
// // "test_training_dropout_zero_ratio_mask",
|
|
|
|
|
// // "test_training_dropout_zero_ratio",
|
|
|
|
|
// // "test_training_dropout",
|
|
|
|
|
"test_transpose_all_permutations_0",
|
|
|
|
|
"test_transpose_all_permutations_1",
|
|
|
|
|
"test_transpose_all_permutations_2",
|
|
|
|
|
"test_transpose_all_permutations_3",
|
|
|
|
|
"test_transpose_all_permutations_4",
|
|
|
|
|
"test_transpose_all_permutations_5",
|
[JS/WebGPU] Unsqueeze operator implementation (#16138)
### Description
This PR adds an implementation of the Squeeze operator to WebGPU JSEP.
The implementation follows the [operator
schema](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Unsqueeze).
To implement the `Unsqueeze` operator in the same fashion as the
`Squeeze`, I added the `ComputeOutputShape()` method to the
`UnsqueezeBase` class and made some slight modifications. Please let me
know if it is a bad idea and if I should move this method to the JS
implementation.
I also uncommented test case lines in the `suite-test-list.jsonc` file
for both Squeeze and Unsqueeze operators following @hariharans29's
[comment](https://github.com/microsoft/onnxruntime/pull/16024#issuecomment-1565113633).
### How was it tested
1. I created a model with only one operator:
```Python
import onnx.helper
node = onnx.helper.make_node(
"Unsqueeze",
inputs=["T", "axes"],
outputs=["y"],
)
graph = onnx.helper.make_graph([node], "test", [onnx.helper.make_tensor_value_info("T", 1, [3, 4, 5]), onnx.helper.make_tensor_value_info("axes", 7, [2])], [onnx.helper.make_tensor_value_info("y", 1, [3, 1, 4, 5, 1])])
onnx.save(onnx.helper.make_model(graph), "unsqueeze.onnx")
```
2. I compiled the runtime using @fs-eire's
[instructions](https://gist.github.com/fs-eire/a55b2c7e10a6864b9602c279b8b75dce).
3. I ran the test models in the browser using this minimal setup:
```HTML
<html>
<script src=".\dist\ort.webgpu.min.js"></script>
<script>
async function run() {
const session = await ort.InferenceSession.create('unsqueeze.onnx', {executionProviders: ['webgpu']});
console.log(session);
const input = new ort.Tensor('float32', new Float32Array(60), [3, 4, 5]);
const dim = new ort.Tensor('int64', [1n, 4n], [2]);
const output = await session.run({ "T": input, "axes": dim });
console.log(output);
}
run();
</script>
</html>
```
### Motivation and Context
Improve operator coverage for WebGPU JSEP.
2023-06-01 19:23:02 +00:00
|
|
|
"test_transpose_default",
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_tril_neg",
|
|
|
|
|
// "test_tril_one_row_neg",
|
|
|
|
|
// "test_tril_out_neg",
|
|
|
|
|
// "test_tril_out_pos",
|
|
|
|
|
// "test_tril_pos",
|
|
|
|
|
// "test_tril_square_neg",
|
|
|
|
|
// "test_tril_square",
|
|
|
|
|
// "test_tril_zero",
|
|
|
|
|
// "test_tril",
|
|
|
|
|
// "test_triu_neg",
|
|
|
|
|
// "test_triu_one_row",
|
|
|
|
|
// "test_triu_out_neg_out",
|
|
|
|
|
// "test_triu_out_pos",
|
|
|
|
|
// "test_triu_pos",
|
|
|
|
|
// "test_triu_square_neg",
|
|
|
|
|
// "test_triu_square",
|
|
|
|
|
// "test_triu_zero",
|
|
|
|
|
// "test_triu",
|
|
|
|
|
// // "test_unique_not_sorted_without_axis",
|
|
|
|
|
// // "test_unique_sorted_with_axis_3d",
|
|
|
|
|
// // "test_unique_sorted_with_axis",
|
|
|
|
|
// // "test_unique_sorted_with_negative_axis",
|
|
|
|
|
// // "test_unique_sorted_without_axis",
|
[JS/WebGPU] Unsqueeze operator implementation (#16138)
### Description
This PR adds an implementation of the Squeeze operator to WebGPU JSEP.
The implementation follows the [operator
schema](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Unsqueeze).
To implement the `Unsqueeze` operator in the same fashion as the
`Squeeze`, I added the `ComputeOutputShape()` method to the
`UnsqueezeBase` class and made some slight modifications. Please let me
know if it is a bad idea and if I should move this method to the JS
implementation.
I also uncommented test case lines in the `suite-test-list.jsonc` file
for both Squeeze and Unsqueeze operators following @hariharans29's
[comment](https://github.com/microsoft/onnxruntime/pull/16024#issuecomment-1565113633).
### How was it tested
1. I created a model with only one operator:
```Python
import onnx.helper
node = onnx.helper.make_node(
"Unsqueeze",
inputs=["T", "axes"],
outputs=["y"],
)
graph = onnx.helper.make_graph([node], "test", [onnx.helper.make_tensor_value_info("T", 1, [3, 4, 5]), onnx.helper.make_tensor_value_info("axes", 7, [2])], [onnx.helper.make_tensor_value_info("y", 1, [3, 1, 4, 5, 1])])
onnx.save(onnx.helper.make_model(graph), "unsqueeze.onnx")
```
2. I compiled the runtime using @fs-eire's
[instructions](https://gist.github.com/fs-eire/a55b2c7e10a6864b9602c279b8b75dce).
3. I ran the test models in the browser using this minimal setup:
```HTML
<html>
<script src=".\dist\ort.webgpu.min.js"></script>
<script>
async function run() {
const session = await ort.InferenceSession.create('unsqueeze.onnx', {executionProviders: ['webgpu']});
console.log(session);
const input = new ort.Tensor('float32', new Float32Array(60), [3, 4, 5]);
const dim = new ort.Tensor('int64', [1n, 4n], [2]);
const output = await session.run({ "T": input, "axes": dim });
console.log(output);
}
run();
</script>
</html>
```
### Motivation and Context
Improve operator coverage for WebGPU JSEP.
2023-06-01 19:23:02 +00:00
|
|
|
"test_unsqueeze_axis_0",
|
|
|
|
|
"test_unsqueeze_axis_1",
|
|
|
|
|
"test_unsqueeze_axis_2",
|
|
|
|
|
"test_unsqueeze_axis_3",
|
|
|
|
|
"test_unsqueeze_negative_axes",
|
|
|
|
|
"test_unsqueeze_three_axes",
|
|
|
|
|
"test_unsqueeze_two_axes",
|
|
|
|
|
"test_unsqueeze_unsorted_axes",
|
2023-09-14 20:14:11 +00:00
|
|
|
"test_unsqueeze",
|
|
|
|
|
"test_wrap_pad"
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
// "test_upsample_nearest",
|
|
|
|
|
// "test_where_example",
|
|
|
|
|
// "test_where_long_example",
|
|
|
|
|
// "test_xor_bcast3v1d",
|
|
|
|
|
// "test_xor_bcast3v2d",
|
|
|
|
|
// "test_xor_bcast4v2d",
|
|
|
|
|
// "test_xor_bcast4v3d",
|
|
|
|
|
// "test_xor_bcast4v4d",
|
|
|
|
|
// "test_xor2d",
|
|
|
|
|
// "test_xor3d",
|
|
|
|
|
// "test_xor4d"
|
|
|
|
|
],
|
2023-07-06 15:35:19 +00:00
|
|
|
"ops": [
|
|
|
|
|
"abs.jsonc",
|
2023-09-06 19:00:16 +00:00
|
|
|
"abs-int32.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"acos.jsonc",
|
|
|
|
|
"add.jsonc",
|
2023-08-18 19:19:01 +00:00
|
|
|
"add_int32.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
//"and.jsonc",
|
|
|
|
|
"asin.jsonc",
|
2023-11-17 20:23:52 +00:00
|
|
|
"attention.jsonc",
|
2023-11-22 23:58:06 +00:00
|
|
|
"batch-norm.jsonc",
|
2023-10-03 19:20:20 +00:00
|
|
|
"bias-add.jsonc",
|
|
|
|
|
"bias-split-gelu.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"ceil.jsonc",
|
2023-09-13 07:05:00 +00:00
|
|
|
"concat.jsonc",
|
|
|
|
|
"concat_int32.jsonc",
|
2023-08-19 06:51:03 +00:00
|
|
|
"cast.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"conv.jsonc",
|
|
|
|
|
"cos.jsonc",
|
|
|
|
|
"div.jsonc",
|
2023-08-18 19:19:01 +00:00
|
|
|
"div_int32.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
//"depth-to-space.jsonc",
|
2023-08-28 02:50:17 +00:00
|
|
|
"equal.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"exp.jsonc",
|
2023-08-16 18:24:26 +00:00
|
|
|
"expand.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"floor.jsonc",
|
2023-08-28 16:55:25 +00:00
|
|
|
"gather-elements.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"gemm.jsonc",
|
|
|
|
|
"global-average-pool.jsonc",
|
2023-08-25 19:11:25 +00:00
|
|
|
"greater.jsonc",
|
[js/webgpu] Optimize InstanceNormalization (#17491)
### Description
<!-- Describe your changes. -->
In previous implementation, there are two loops to iterate H * W
elements to calculate the `mean` and `squaredNorm` value in one thread,
meanwhile it outputs H * W elements in one thread. That results it's
very very slow when H * W is a large value. And usually, H * W does be a
large value in a model. For example, in the `candy-8` model, the shapes
of [H, W] are [224,224], [112,112], [56,56] for `InstanceNormalization`
op. And in my ADL, `[1,224,224,32]` consumes 17 ms. See below:
```
[profiling] kernel "23848328|[InstanceNormalization] 23848328" input[0]: [1,224,224,32] | float32, input[1]: [32] | float32, input[2]: [32] | float32, output[0]: [1,224,224,32] | float32, execution time: 17007914 ns
```
In this PR, it uses workgroup memory to optimize the original algorithm.
The advantage is that it can parallelly utilize the 64 (workgroupSize)
threads in one workgroup to calculate `mean` and `squaredNorm` value.
Meanwhile, it only outputs `H * W / workgroupSize` outputs for one
thread, which greatly reduces the overhead for one thread. With this
optimization, `[1,224,224,32]` becomes 3 ms and the main overhead is the
extra two `transpose`. The `createInstanceNormProgramInfo` only needs
`0.64` ms. See below:
```
[profiling] kernel "23003600|[InstanceNormalization] 23003600" input[0]: [1,224,224,32] | float32, output[0]: [1,32,224,224] | float32, execution time: 1543792 ns
program-manager.ts:115
[profiling] kernel "23003600|[InstanceNormalization] 23003600" input[0]: [1,32,224,224] | float32, input[1]: [32] | float32, input[2]: [32] | float32, output[0]: [1,32,224,224] | float32, execution time: 642652 ns
program-manager.ts:115
[profiling] kernel "23003600|[InstanceNormalization] 23003600" input[0]: [1,32,224,224] | float32, output[0]: [1,224,224,32] | float32, execution time: 991608 ns
```
This PR currently only applies the new algorithm to NCHW format. For
NHWC format, one way is to transpose the input so that it can use the
new algorithm. But the disadvantage is that 2 extra transpose are added.
@dakenf also gives another way to optimize NHWC. Details see
[here](https://github.com/microsoft/onnxruntime/blob/d45a96616da9843b037210f2d48d6b4e5bdae5c6/js/web/lib/wasm/jsep/webgpu/ops/instance-norm.ts).
I checked @dakenf's method. The perf is similar with transpose +
optimized NCHW. But on different GPUs, one is a little better than
another or vice versa. So I prefer this PR only does the NCHW part.
@dakenf can submit his optimization on NHWC.
2023-09-15 00:03:18 +00:00
|
|
|
"instance-norm.jsonc",
|
2023-08-25 19:11:25 +00:00
|
|
|
"less.jsonc",
|
2023-08-15 01:04:12 +00:00
|
|
|
"log.jsonc",
|
2023-08-29 19:40:57 +00:00
|
|
|
"matmul.jsonc",
|
2023-09-06 03:41:46 +00:00
|
|
|
"matmul-broadcast.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"mul.jsonc",
|
2023-08-18 19:19:01 +00:00
|
|
|
"mul_int32.jsonc",
|
2023-11-17 20:23:52 +00:00
|
|
|
"multi-head-attention.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
//"neg.jsonc",
|
2023-09-06 19:00:16 +00:00
|
|
|
"neg-int32.jsonc",
|
2023-08-28 02:50:17 +00:00
|
|
|
"not.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
//"or.jsonc",
|
2023-08-16 17:41:20 +00:00
|
|
|
"layer-norm.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"leaky-relu.jsonc",
|
|
|
|
|
"reduce-min.jsonc",
|
|
|
|
|
"relu.jsonc",
|
2023-07-31 16:18:58 +00:00
|
|
|
"gelu.jsonc",
|
2023-09-14 20:14:11 +00:00
|
|
|
"pad.jsonc",
|
|
|
|
|
"pad-big.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"pow.jsonc",
|
2023-08-18 19:19:01 +00:00
|
|
|
"pow_int32.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"pow-big-number.jsonc",
|
|
|
|
|
"reshape.jsonc",
|
2023-08-18 16:59:03 +00:00
|
|
|
"skip-layer-norm.jsonc",
|
2023-09-06 01:05:47 +00:00
|
|
|
"slice.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
//"softmax.jsonc",
|
|
|
|
|
"sin.jsonc",
|
|
|
|
|
//"split.jsonc",
|
|
|
|
|
"sqrt.jsonc",
|
|
|
|
|
"sub.jsonc",
|
2023-08-18 19:19:01 +00:00
|
|
|
"sub_int32.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
"tan.jsonc",
|
2023-08-18 17:07:21 +00:00
|
|
|
"tile.jsonc",
|
2023-08-02 23:27:24 +00:00
|
|
|
"transpose.jsonc",
|
2023-10-03 21:28:21 +00:00
|
|
|
"transpose_int32_uint32.jsonc",
|
2023-12-28 00:21:29 +00:00
|
|
|
"upsample.jsonc",
|
2023-10-03 21:28:21 +00:00
|
|
|
"where.jsonc"
|
|
|
|
|
// Turn on this when https://github.com/microsoft/onnxruntime/issues/17405 is fixed.
|
|
|
|
|
//"where_broadcast.jsonc",
|
2023-07-06 15:35:19 +00:00
|
|
|
//"xor.jsonc"
|
|
|
|
|
]
|
[js/web] WebGPU backend via JSEP (#14579)
### Description
This change introduced the following new components into ONNX Runtime
Web:
- JavaScript Execution Provider (JSEP)
- Asynchronized inferencing execution powered by Emscripten's Asyncify
- WebGPU backend implemented in TypeScript
- initial implementation of kernels:
- elementwise operators (22)
- binary operators (5)
- tensor: Shape, Reshape, Transpose, Gemm
- nn: Conv, {Global}Maxpool, {Global}AveragePool
Code need to be polished. still working on it.
## Q&A
What is JSEP?
> JSEP, aka JavaScript Execution Provider, is a new ONNXRuntime
execution provider that specifically works on Web environment
(browsers). JSEP allows JavaScript code to kick in from various places
when ONNX Runtime inferences a model.
Why JSEP?
> JSEP is a hybrid mode EP that contains both C/C++ and
TypeScript/JavaScript implementation. There are 2 strong reasons why we
introduces JSEP:
> 1. the C/C++ part helps JSEP to leverage ONNX Runtime's capabilities
as much as possible including graph transformer, optimizers and also the
capabilities to fallback to CPU EP. TypeScript/JavaScript helps JSEP to
develop and debug much easier in the browser for the kernel
implementation.
> 2. the requirement of asynchronized execution from JavaScript API (eg.
`buffer.mapAsync()`) makes it impossible to run `OrtRun()` in a
synchronized context (see "async problem" section below). This is done
by using Emscripten's Asyncify.
What is WebGPU?
> WebGPU is the new GPU API that available in browser. It's one of the
only 2 APIs that currently available to access the GPU from browser (the
other is WebGL).
> WebGPU is designed with more advanced and stronger features comparing
to WebGL and is potentially solution that offer the best GPU performance
for model inferencing that currently available.
What is the async problem and why we have the problem?
> The "async problem" is a problem that you cannot call an async
function in a synchronous context. Think about the following C++ code:
> ```c
> // C-style declarations (API)
> typedef void (*ON_COMPLETE)(PVOID state, DATA *data);
> void read_data_from_file(FILEHANDLE file, ON_COMPLETE on_complete);
>
> // implementation
> DATA * my_impl_read_data_from_file_sync(FILEHANDLE file) {
> // how to implement?
> }
> ```
> The answer is, it's impossible to implement this function. Usually we
try to find a sync version API, or launch a thread to call the async
function and sync-wait on the main thread. Unfortunately, in browser
environment, neither is possible.
>
> WebGPU does not offer any synchronized API for data downloading (GPU
to CPU). This is the only operation that MUST be async. As `OrtRun()`
will eventually call into DataTransfer for copy data from GPU to CPU,
and `OrtRun()` is a synchronized function, this cannot be done in normal
way.
What is Emscripten? How is the Asyncify feature resolved the problem?
> Emscripten is the C/C++ compiler for WebAssembly. It's what we use to
compile ORT and generates the WebAssembly artifacts which runs on
browsers.
>
> Asyncify is a [compiler
feature](https://emscripten.org/docs/porting/asyncify.html) that allows
calling async functions from a synchronized context. In short, it
generates code to unwind and rewind call stack to emulate async
execution. With this feature, we are able to call the async function
inside `OrtRun()` call.
## Design Overview
**Inter-op**
JSEP is doing pretty much same thing to just another EP. It exposes an
interface for inter-op with JavaScript, which is defined in
onnxruntime/wasm/js_internal_api.js:
```js
// init JSEP
Module["jsepInit"] = function (backend, alloc, free, copy, copyAsync, createKernel, releaseKernel, run) {
Module.jsepBackend = backend;
Module.jsepAlloc = alloc;
Module.jsepFree = free;
Module.jsepCopy = copy;
Module.jsepCopyAsync = copyAsync;
Module.jsepCreateKernel = createKernel;
Module.jsepReleaseKernel = releaseKernel;
Module.jsepRun = run;
};
```
This simple JavaScript snippet defines all language barrier level
functions that requires by JSEP to achieve implementing kernels and data
transfers using JavaScript inside ONNX Runtime:
- `jsepBackend`: assign the singleton object to webassembly module
- `jsepAlloc` and `jsepFree`: implementation of data transfer's Alloc()
and Free()
- `jsepCopy`: synchronized copy ( GPU to GPU, CPU to GPU)
- `jsepCopyAsync`: asynchronized copy ( GPU to CPU)
- `jsepCreateKernel` and `jsepReleaseKernel`: a corresponding object
that maintained in JS to match lifecycle of Kernel in ORT
- `jsepRun`: OpKernel::Compute() should call into this
The abstraction above allows to tie as little as possible connections
and dependencies between C/C++ and TypeScript/JavaScript.
**Resource Management**
Lifecycle of tensor data and kernels are managed by ORT(C/C++) but the
implementation are left to JavaScript. JavaScript code are responsible
to implement the callbacks correctly.
For WebGPU, the GPU data is managed by JavaScript using a singleton map
(tensot_data_id => GPUBuffer). GPU pipeline is managed as singleton.
Shaders are managed using a singletonmap (shader_key => gpu_program),
while shader_key is generated by cache_key (OP specific, including
attributes) and input shapes.
**about data transfer**
`js::DataTransfer::CopyTensor` implemented to call either synchronized
or asynchronized copy callback, depending on the destination is GPU or
not. Emscripten's macro `EM_ASYNC_JS` is used to wrap the async function
to be called in the synchronized context.
**run kernel in JS**
Kernel class constructor calls once `jsepCreateKernel()` with an
optional per-kernel specific serialization to pass attributes into
JavaScript.
`Compute()` are implemented in a way that a metadata serialization is
performed in a base class and JavaScript code can access the data using
the Emscripten specific builtin macro `EM_ASM_*`.
**disabled features**
memory pattern is force disabled, because the WebGPU data is not
presented by a general memory model (a buffer can be represented by
offset + size).
concurrent run support is disabled. WebGPU is stateful and it also has
async function call. To support concurrent run will significantly
increase the complexity and we don't get any real benefit from it.
**prefer channels last**
JSEP prefers channels last and returns `DataLayout::NHWC` in method
`GetPreferredLayout()`. This will let the graph transformers to
preprocess the graph into a channels last form so that a more optimized
WebGPU shader can be used.
**Testing code**
It's impossible to test JSEP directly because JSEP itself does not
contain any kernel implementation. However, it has the kernel
registration which need to work together with the corresponding
JavaScript code. There are unit tests that run onnx models from
JavaScript API.
---------
Co-authored-by: Scott McKay <skottmckay@gmail.com>
2023-04-24 22:21:18 +00:00
|
|
|
},
|
2021-04-27 07:04:25 +00:00
|
|
|
"wasm": {
|
2023-08-01 08:10:55 +00:00
|
|
|
"onnx": ["resnet50", "squeezenet", "tiny_yolov2", "emotion_ferplus"],
|
2021-04-27 07:04:25 +00:00
|
|
|
"node": [
|
|
|
|
|
// Check in node tests that have native Wasm implementations
|
|
|
|
|
// (i.e.) not tests that rely on the fallback cpu implementations
|
|
|
|
|
// Use the 'cpu' level of node tests to test those implementations
|
|
|
|
|
"test_add_bcast",
|
|
|
|
|
"test_add",
|
|
|
|
|
"test_sub_bcast",
|
|
|
|
|
"test_sub_example",
|
|
|
|
|
"test_sub",
|
|
|
|
|
"test_mul_bcast",
|
|
|
|
|
"test_mul_example",
|
|
|
|
|
"test_mul",
|
|
|
|
|
"test_div_bcast",
|
|
|
|
|
"test_div_example",
|
|
|
|
|
"test_div",
|
|
|
|
|
"test_xor_bcast3v1d",
|
|
|
|
|
"test_xor_bcast3v2d",
|
|
|
|
|
"test_xor_bcast4v2d",
|
|
|
|
|
"test_xor_bcast4v3d",
|
|
|
|
|
"test_xor_bcast4v4d",
|
|
|
|
|
"test_xor2d",
|
|
|
|
|
"test_xor3d",
|
|
|
|
|
"test_xor4d",
|
|
|
|
|
"test_or_bcast3v1d",
|
|
|
|
|
"test_or_bcast3v2d",
|
|
|
|
|
"test_or_bcast4v2d",
|
|
|
|
|
"test_or_bcast4v3d",
|
|
|
|
|
"test_or_bcast4v4d",
|
|
|
|
|
"test_and_bcast3v1d",
|
|
|
|
|
"test_and_bcast3v2d",
|
|
|
|
|
"test_and_bcast4v2d",
|
|
|
|
|
"test_and_bcast4v3d",
|
|
|
|
|
"test_and_bcast4v4d",
|
|
|
|
|
"test_and2d",
|
|
|
|
|
"test_and3d",
|
|
|
|
|
"test_and4d",
|
2023-05-09 04:25:10 +00:00
|
|
|
"test_prelu_broadcast",
|
|
|
|
|
"test_prelu_example",
|
|
|
|
|
"test_basic_conv_with_padding",
|
|
|
|
|
"test_basic_conv_without_padding",
|
|
|
|
|
"test_batchnorm_epsilon",
|
|
|
|
|
"test_batchnorm_example",
|
|
|
|
|
"opset{10,11,12}/test_cast_STRING_to_FLOAT",
|
|
|
|
|
"test_clip_splitbounds",
|
|
|
|
|
"test_clip_outbounds",
|
|
|
|
|
"test_clip_inbounds",
|
|
|
|
|
"test_clip_example",
|
|
|
|
|
"test_clip_default_min",
|
|
|
|
|
"test_clip_default_max",
|
|
|
|
|
"test_clip_default_inbounds",
|
|
|
|
|
"test_clip",
|
|
|
|
|
"test_conv_with_strides_and_asymmetric_padding",
|
|
|
|
|
"test_conv_with_strides_no_padding",
|
|
|
|
|
"test_conv_with_strides_padding",
|
|
|
|
|
"test_gemm_nobroadcast",
|
|
|
|
|
"test_gemm_broadcast",
|
|
|
|
|
"test_matmul_2d",
|
|
|
|
|
"test_matmul_3d",
|
|
|
|
|
"test_matmul_4d",
|
|
|
|
|
"test_softmax_axis_0",
|
|
|
|
|
"test_softmax_axis_1",
|
|
|
|
|
"test_softmax_axis_2",
|
|
|
|
|
"test_softmax_default_axis",
|
|
|
|
|
"test_softmax_example",
|
|
|
|
|
"test_softmax_large_number",
|
|
|
|
|
"test_sum_example",
|
|
|
|
|
"test_sum_one_input",
|
|
|
|
|
"test_sum_two_inputs",
|
|
|
|
|
"test_averagepool_1d_default",
|
|
|
|
|
"test_averagepool_2d_default",
|
|
|
|
|
"test_averagepool_2d_pads",
|
|
|
|
|
"test_averagepool_2d_precomputed_pads",
|
|
|
|
|
"test_averagepool_2d_precomputed_same_upper",
|
|
|
|
|
"test_averagepool_2d_precomputed_strides",
|
|
|
|
|
"test_averagepool_2d_same_upper",
|
|
|
|
|
"test_averagepool_2d_same_lower",
|
|
|
|
|
"test_averagepool_2d_strides",
|
|
|
|
|
"test_averagepool_3d_default",
|
|
|
|
|
"test_maxpool_1d_default",
|
|
|
|
|
"test_maxpool_2d_default",
|
|
|
|
|
"test_maxpool_2d_pads",
|
|
|
|
|
"test_maxpool_2d_precomputed_pads",
|
|
|
|
|
"test_maxpool_2d_precomputed_same_upper",
|
|
|
|
|
"test_maxpool_2d_precomputed_strides",
|
|
|
|
|
"test_maxpool_2d_same_lower",
|
|
|
|
|
"test_maxpool_2d_same_upper",
|
|
|
|
|
"test_maxpool_2d_strides",
|
|
|
|
|
"test_maxpool_3d_default",
|
|
|
|
|
"test_globalaveragepool_precomputed",
|
|
|
|
|
"test_globalaveragepool",
|
|
|
|
|
"test_globalmaxpool_precomputed",
|
|
|
|
|
"test_globalmaxpool",
|
|
|
|
|
"test_instancenorm_epsilon",
|
|
|
|
|
"test_instancenorm_example"
|
|
|
|
|
],
|
|
|
|
|
"ops": []
|
|
|
|
|
},
|
|
|
|
|
"webnn": {
|
2023-08-01 08:10:55 +00:00
|
|
|
"onnx": ["resnet50", "squeezenet", "tiny_yolov2", "emotion_ferplus"],
|
2023-05-09 04:25:10 +00:00
|
|
|
"node": [
|
2024-01-09 18:10:57 +00:00
|
|
|
"test_abs",
|
|
|
|
|
"test_acos_example",
|
|
|
|
|
"test_acos",
|
|
|
|
|
"test_acosh_example",
|
|
|
|
|
"test_acosh",
|
|
|
|
|
// // "test_adagrad_multiple",
|
|
|
|
|
// // "test_adagrad",
|
|
|
|
|
// // "test_adam_multiple",
|
|
|
|
|
// // "test_adam",
|
2023-05-09 04:25:10 +00:00
|
|
|
"test_add_bcast",
|
2024-01-09 18:10:57 +00:00
|
|
|
// "test_add_uint8",
|
2023-05-09 04:25:10 +00:00
|
|
|
"test_add",
|
2024-01-09 18:10:57 +00:00
|
|
|
// "test_and_bcast3v1d",
|
|
|
|
|
// "test_and_bcast3v2d",
|
|
|
|
|
// "test_and_bcast4v2d",
|
|
|
|
|
// "test_and_bcast4v3d",
|
|
|
|
|
// "test_and_bcast4v4d",
|
|
|
|
|
// "test_and2d",
|
|
|
|
|
// "test_and3d",
|
|
|
|
|
// "test_and4d",
|
|
|
|
|
// "test_argmax_default_axis_example_select_last_index",
|
|
|
|
|
// "test_argmax_default_axis_example",
|
|
|
|
|
// "test_argmax_default_axis_random_select_last_index",
|
|
|
|
|
// "test_argmax_default_axis_random",
|
|
|
|
|
// "test_argmax_keepdims_example_select_last_index",
|
|
|
|
|
// "test_argmax_keepdims_example",
|
|
|
|
|
// "test_argmax_keepdims_random_select_last_index",
|
|
|
|
|
// "test_argmax_keepdims_random",
|
|
|
|
|
// "test_argmax_negative_axis_keepdims_example_select_last_index",
|
|
|
|
|
// "test_argmax_negative_axis_keepdims_example",
|
|
|
|
|
// "test_argmax_negative_axis_keepdims_random_select_last_index",
|
|
|
|
|
// "test_argmax_negative_axis_keepdims_random",
|
|
|
|
|
// "test_argmax_no_keepdims_example_select_last_index",
|
|
|
|
|
// "test_argmax_no_keepdims_example",
|
|
|
|
|
// "test_argmax_no_keepdims_random_select_last_index",
|
|
|
|
|
// "test_argmax_no_keepdims_random",
|
|
|
|
|
// "test_argmin_default_axis_example_select_last_index",
|
|
|
|
|
// "test_argmin_default_axis_example",
|
|
|
|
|
// "test_argmin_default_axis_random_select_last_index",
|
|
|
|
|
// "test_argmin_default_axis_random",
|
|
|
|
|
// "test_argmin_keepdims_example_select_last_index",
|
|
|
|
|
// "test_argmin_keepdims_example",
|
|
|
|
|
// "test_argmin_keepdims_random_select_last_index",
|
|
|
|
|
// "test_argmin_keepdims_random",
|
|
|
|
|
// "test_argmin_negative_axis_keepdims_example_select_last_index",
|
|
|
|
|
// "test_argmin_negative_axis_keepdims_example",
|
|
|
|
|
// "test_argmin_negative_axis_keepdims_random_select_last_index",
|
|
|
|
|
// "test_argmin_negative_axis_keepdims_random",
|
|
|
|
|
// "test_argmin_no_keepdims_example_select_last_index",
|
|
|
|
|
// "test_argmin_no_keepdims_example",
|
|
|
|
|
// "test_argmin_no_keepdims_random_select_last_index",
|
|
|
|
|
// "test_argmin_no_keepdims_random",
|
|
|
|
|
// "test_asin_example",
|
|
|
|
|
// "test_asin",
|
|
|
|
|
// "test_asinh_example",
|
|
|
|
|
// "test_asinh",
|
|
|
|
|
// "test_atan_example",
|
|
|
|
|
// "test_atan",
|
|
|
|
|
// "test_atanh_example",
|
|
|
|
|
// "test_atanh",
|
|
|
|
|
// "test_averagepool_1d_default",
|
|
|
|
|
// "test_averagepool_2d_ceil",
|
|
|
|
|
"test_averagepool_2d_default",
|
|
|
|
|
"test_averagepool_2d_pads_count_include_pad",
|
|
|
|
|
"test_averagepool_2d_pads",
|
|
|
|
|
"test_averagepool_2d_precomputed_pads_count_include_pad",
|
|
|
|
|
"test_averagepool_2d_precomputed_pads",
|
|
|
|
|
"test_averagepool_2d_precomputed_same_upper",
|
|
|
|
|
"test_averagepool_2d_precomputed_strides",
|
|
|
|
|
"test_averagepool_2d_same_lower",
|
|
|
|
|
"test_averagepool_2d_same_upper",
|
|
|
|
|
"test_averagepool_2d_strides",
|
|
|
|
|
// "test_averagepool_3d_default",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_basic_conv_with_padding",
|
|
|
|
|
"test_basic_conv_without_padding",
|
2024-01-09 18:10:57 +00:00
|
|
|
// "test_basic_convinteger",
|
|
|
|
|
// "test_batchnorm_epsilon_training_mode",
|
|
|
|
|
// "test_batchnorm_epsilon",
|
|
|
|
|
// "test_batchnorm_example_training_mode",
|
|
|
|
|
// "test_batchnorm_example",
|
|
|
|
|
// // "test_bernoulli_double_expanded",
|
|
|
|
|
// // "test_bernoulli_double",
|
|
|
|
|
// // "test_bernoulli_expanded",
|
|
|
|
|
// // "test_bernoulli_seed_expanded",
|
|
|
|
|
// // "test_bernoulli_seed",
|
|
|
|
|
// // "test_bernoulli",
|
|
|
|
|
// // "test_bitshift_left_uint16",
|
|
|
|
|
// // "test_bitshift_left_uint32",
|
|
|
|
|
// // "test_bitshift_left_uint64",
|
|
|
|
|
// // "test_bitshift_left_uint8",
|
|
|
|
|
// // "test_bitshift_right_uint16",
|
|
|
|
|
// // "test_bitshift_right_uint32",
|
|
|
|
|
// // "test_bitshift_right_uint64",
|
|
|
|
|
// // "test_bitshift_right_uint8",
|
|
|
|
|
// // "test_blackmanwindow_expanded",
|
|
|
|
|
// // "test_blackmanwindow_symmetric_expanded",
|
|
|
|
|
// // "test_blackmanwindow_symmetric",
|
|
|
|
|
// // "test_blackmanwindow",
|
|
|
|
|
// // "test_cast_BFLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_cast_DOUBLE_to_FLOAT",
|
|
|
|
|
// // "test_cast_DOUBLE_to_FLOAT16",
|
|
|
|
|
// // "test_cast_FLOAT_to_BFLOAT16",
|
|
|
|
|
// // "test_cast_FLOAT_to_DOUBLE",
|
|
|
|
|
// // "test_cast_FLOAT_to_FLOAT16",
|
|
|
|
|
// // "test_cast_FLOAT_to_STRING",
|
|
|
|
|
// // "test_cast_FLOAT16_to_DOUBLE",
|
|
|
|
|
// // "test_cast_FLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_cast_STRING_to_FLOAT",
|
|
|
|
|
// // "test_castlike_BFLOAT16_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_BFLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT16_expanded",
|
|
|
|
|
// // "test_castlike_DOUBLE_to_FLOAT16",
|
|
|
|
|
// // "test_castlike_FLOAT_to_BFLOAT16_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_BFLOAT16",
|
|
|
|
|
// // "test_castlike_FLOAT_to_DOUBLE_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_DOUBLE",
|
|
|
|
|
// // "test_castlike_FLOAT_to_FLOAT16_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_FLOAT16",
|
|
|
|
|
// // "test_castlike_FLOAT_to_STRING_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT_to_STRING",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_DOUBLE_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_DOUBLE",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_FLOAT16_to_FLOAT",
|
|
|
|
|
// // "test_castlike_STRING_to_FLOAT_expanded",
|
|
|
|
|
// // "test_castlike_STRING_to_FLOAT",
|
|
|
|
|
"test_ceil_example",
|
|
|
|
|
"test_ceil",
|
|
|
|
|
// "test_celu_expanded",
|
|
|
|
|
// "test_celu",
|
2021-10-14 23:29:37 +00:00
|
|
|
"test_clip_default_inbounds",
|
2024-01-09 18:10:57 +00:00
|
|
|
"test_clip_default_int8_inbounds",
|
|
|
|
|
"test_clip_default_int8_max",
|
|
|
|
|
"test_clip_default_int8_min",
|
|
|
|
|
"test_clip_default_max",
|
|
|
|
|
"test_clip_default_min",
|
|
|
|
|
"test_clip_example",
|
|
|
|
|
"test_clip_inbounds",
|
|
|
|
|
"test_clip_outbounds",
|
|
|
|
|
"test_clip_splitbounds",
|
2021-10-14 23:29:37 +00:00
|
|
|
"test_clip",
|
2024-01-09 18:10:57 +00:00
|
|
|
// // "test_compress_0",
|
|
|
|
|
// // "test_compress_1",
|
|
|
|
|
// // "test_compress_default_axis",
|
|
|
|
|
// // "test_compress_negative_axis",
|
|
|
|
|
"test_concat_1d_axis_0",
|
|
|
|
|
"test_concat_1d_axis_negative_1",
|
|
|
|
|
"test_concat_2d_axis_0",
|
|
|
|
|
"test_concat_2d_axis_1",
|
|
|
|
|
"test_concat_2d_axis_negative_1",
|
|
|
|
|
"test_concat_2d_axis_negative_2",
|
|
|
|
|
"test_concat_3d_axis_0",
|
|
|
|
|
"test_concat_3d_axis_1",
|
|
|
|
|
"test_concat_3d_axis_2",
|
|
|
|
|
"test_concat_3d_axis_negative_1",
|
|
|
|
|
"test_concat_3d_axis_negative_2",
|
|
|
|
|
"test_concat_3d_axis_negative_3",
|
|
|
|
|
"test_conv_with_autopad_same",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_conv_with_strides_and_asymmetric_padding",
|
|
|
|
|
"test_conv_with_strides_no_padding",
|
|
|
|
|
"test_conv_with_strides_padding",
|
2024-01-09 18:10:57 +00:00
|
|
|
// // "test_convinteger_with_padding",
|
|
|
|
|
// // "test_convinteger_without_padding",
|
|
|
|
|
// "test_convtranspose_1d",
|
|
|
|
|
// // "test_convtranspose_3d",
|
|
|
|
|
// "test_convtranspose_autopad_same",
|
|
|
|
|
"test_convtranspose_dilations",
|
|
|
|
|
"test_convtranspose_kernel_shape",
|
|
|
|
|
"opset{9,17}/test_convtranspose_output_shape",
|
|
|
|
|
"test_convtranspose_pad",
|
|
|
|
|
"test_convtranspose_pads",
|
|
|
|
|
"test_convtranspose_with_kernel",
|
|
|
|
|
"test_convtranspose",
|
|
|
|
|
"test_cos_example",
|
|
|
|
|
"test_cos",
|
|
|
|
|
// "test_cosh_example",
|
|
|
|
|
// "test_cosh",
|
|
|
|
|
// "test_cumsum_1d_exclusive",
|
|
|
|
|
// "test_cumsum_1d_reverse_exclusive",
|
|
|
|
|
// "test_cumsum_1d_reverse",
|
|
|
|
|
// "test_cumsum_1d",
|
|
|
|
|
// "test_cumsum_2d_axis_0",
|
|
|
|
|
// "test_cumsum_2d_axis_1",
|
|
|
|
|
// "test_cumsum_2d_negative_axis",
|
|
|
|
|
// "test_depthtospace_crd_mode_example",
|
|
|
|
|
// "test_depthtospace_crd_mode",
|
|
|
|
|
// "test_depthtospace_dcr_mode",
|
|
|
|
|
// "test_depthtospace_example",
|
|
|
|
|
// "test_depthtospace",
|
|
|
|
|
// // "test_dequantizelinear_axis",
|
|
|
|
|
// // "test_dequantizelinear",
|
|
|
|
|
// // "test_det_2d",
|
|
|
|
|
// // "test_det_nd",
|
|
|
|
|
// // "test_dft_axis",
|
|
|
|
|
// // "test_dft_inverse",
|
|
|
|
|
// // "test_dft",
|
|
|
|
|
"test_div_bcast",
|
|
|
|
|
"test_div_example",
|
|
|
|
|
// "test_div_uint8",
|
|
|
|
|
"test_div",
|
|
|
|
|
// // "test_dropout_default_mask_ratio",
|
|
|
|
|
// // "test_dropout_default_mask",
|
|
|
|
|
// // "test_dropout_default_old",
|
|
|
|
|
// // "test_dropout_default_ratio",
|
|
|
|
|
// // "test_dropout_default",
|
|
|
|
|
// // "test_dropout_random_old",
|
|
|
|
|
// // "test_dropout_random",
|
|
|
|
|
// // "test_dynamic_slice_default_axes",
|
|
|
|
|
// // "test_dynamic_slice_end_out_of_bounds",
|
|
|
|
|
// // "test_dynamic_slice_neg",
|
|
|
|
|
// // "test_dynamic_slice_start_out_of_bounds",
|
|
|
|
|
// // "test_dynamic_slice",
|
|
|
|
|
// // "test_dynamicquantizelinear_expanded",
|
|
|
|
|
// // "test_dynamicquantizelinear_max_adjusted_expanded",
|
|
|
|
|
// // "test_dynamicquantizelinear_max_adjusted",
|
|
|
|
|
// // "test_dynamicquantizelinear_min_adjusted_expanded",
|
|
|
|
|
// // "test_dynamicquantizelinear_min_adjusted",
|
|
|
|
|
// // "test_dynamicquantizelinear",
|
|
|
|
|
// "test_edge_pad",
|
|
|
|
|
// "test_einsum_batch_diagonal",
|
|
|
|
|
// "test_einsum_batch_matmul",
|
|
|
|
|
// "test_einsum_inner_prod",
|
|
|
|
|
// "test_einsum_sum",
|
|
|
|
|
// "test_einsum_transpose",
|
|
|
|
|
"test_elu_default",
|
|
|
|
|
"test_elu_example",
|
|
|
|
|
"test_elu",
|
|
|
|
|
// "test_equal_bcast",
|
|
|
|
|
// "test_equal",
|
|
|
|
|
// "test_erf",
|
|
|
|
|
"test_exp_example",
|
|
|
|
|
"test_exp",
|
|
|
|
|
// "test_expand_dim_changed",
|
|
|
|
|
// "test_expand_dim_unchanged",
|
|
|
|
|
// "test_eyelike_populate_off_main_diagonal",
|
|
|
|
|
// "test_eyelike_with_dtype",
|
|
|
|
|
// "test_eyelike_without_dtype",
|
|
|
|
|
"test_flatten_axis0",
|
|
|
|
|
"test_flatten_axis1",
|
|
|
|
|
"test_flatten_axis2",
|
|
|
|
|
"test_flatten_axis3",
|
|
|
|
|
"test_flatten_default_axis",
|
|
|
|
|
"test_flatten_negative_axis1",
|
|
|
|
|
"test_flatten_negative_axis2",
|
|
|
|
|
"test_flatten_negative_axis3",
|
|
|
|
|
"test_flatten_negative_axis4",
|
|
|
|
|
"test_floor_example",
|
|
|
|
|
"test_floor",
|
|
|
|
|
// "test_gather_0",
|
|
|
|
|
// "test_gather_1",
|
|
|
|
|
// "test_gather_2d_indices",
|
|
|
|
|
// "test_gather_negative_indices",
|
|
|
|
|
// "test_gather_elements_0",
|
|
|
|
|
// "test_gather_elements_1",
|
|
|
|
|
// "test_gather_elements_negative_indices",
|
|
|
|
|
// "test_gather_negative_indices",
|
|
|
|
|
// "test_gathernd_example_float32",
|
|
|
|
|
// "test_gathernd_example_int32_batch_dim1",
|
|
|
|
|
// "test_gathernd_example_int32",
|
|
|
|
|
"test_gemm_all_attributes",
|
|
|
|
|
"test_gemm_alpha",
|
|
|
|
|
"test_gemm_beta",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_gemm_broadcast",
|
2024-01-09 18:10:57 +00:00
|
|
|
"test_gemm_default_matrix_bias",
|
|
|
|
|
"test_gemm_default_no_bias",
|
|
|
|
|
// "test_gemm_default_scalar_bias",
|
|
|
|
|
"test_gemm_default_single_elem_vector_bias",
|
|
|
|
|
"test_gemm_default_vector_bias",
|
|
|
|
|
"test_gemm_default_zero_bias",
|
|
|
|
|
"test_gemm_nobroadcast",
|
|
|
|
|
"test_gemm_transposeA",
|
|
|
|
|
"test_gemm_transposeB",
|
|
|
|
|
"test_globalaveragepool_precomputed",
|
|
|
|
|
"test_globalaveragepool",
|
|
|
|
|
// "test_globalmaxpool_precomputed",
|
|
|
|
|
// "test_globalmaxpool",
|
|
|
|
|
// "test_greater_bcast",
|
|
|
|
|
// "test_greater_equal_bcast_expanded",
|
|
|
|
|
// "test_greater_equal_bcast",
|
|
|
|
|
// "test_greater_equal_expanded",
|
|
|
|
|
// "test_greater_equal",
|
|
|
|
|
// "test_greater",
|
|
|
|
|
// // "test_gridsample_aligncorners_true",
|
|
|
|
|
// // "test_gridsample_bicubic",
|
|
|
|
|
// // "test_gridsample_bilinear",
|
|
|
|
|
// // "test_gridsample_border_padding",
|
|
|
|
|
// // "test_gridsample_nearest",
|
|
|
|
|
// // "test_gridsample_reflection_padding",
|
|
|
|
|
// // "test_gridsample_zeros_padding",
|
|
|
|
|
// // "test_gridsample",
|
|
|
|
|
// // "test_gru_batchwise",
|
|
|
|
|
// // "test_gru_defaults",
|
|
|
|
|
// // "test_gru_seq_length",
|
|
|
|
|
// // "test_gru_with_initial_bias",
|
|
|
|
|
// // "test_hammingwindow_expanded",
|
|
|
|
|
// // "test_hammingwindow_symmetric_expanded",
|
|
|
|
|
// // "test_hammingwindow_symmetric",
|
|
|
|
|
// // "test_hammingwindow",
|
|
|
|
|
// // "test_hannwindow_expanded",
|
|
|
|
|
// // "test_hannwindow_symmetric_expanded",
|
|
|
|
|
// // "test_hannwindow_symmetric",
|
|
|
|
|
// // "test_hannwindow",
|
|
|
|
|
// // "test_hardmax_axis_0",
|
|
|
|
|
// // "test_hardmax_axis_1",
|
|
|
|
|
// // "test_hardmax_axis_2",
|
|
|
|
|
// // "test_hardmax_default_axis",
|
|
|
|
|
// // "test_hardmax_example",
|
|
|
|
|
// // "test_hardmax_negative_axis",
|
|
|
|
|
// // "test_hardmax_one_hot",
|
|
|
|
|
// // "test_hardsigmoid_default",
|
|
|
|
|
// // "test_hardsigmoid_example",
|
|
|
|
|
// // "test_hardsigmoid",
|
|
|
|
|
// "test_hardswish_expanded",
|
|
|
|
|
"test_hardswish",
|
|
|
|
|
// "test_if",
|
|
|
|
|
// TODO: Uncomment 'test_if_seq' and 'test_if_opt' once the test infra
|
|
|
|
|
// supports Sequence and Optional types
|
|
|
|
|
// "test_if_seq",
|
|
|
|
|
// "test_if_opt",
|
|
|
|
|
"test_instancenorm_epsilon",
|
|
|
|
|
// "test_instancenorm_example",
|
|
|
|
|
// "test_isinf_negative",
|
|
|
|
|
// "test_isinf_positive",
|
|
|
|
|
// "test_isinf",
|
|
|
|
|
// "test_isnan",
|
|
|
|
|
// "test_layer_normalization_2d_axis_negative_1_expanded",
|
|
|
|
|
// "test_layer_normalization_2d_axis_negative_1",
|
|
|
|
|
// "test_layer_normalization_2d_axis_negative_2_expanded",
|
|
|
|
|
// "test_layer_normalization_2d_axis_negative_2",
|
|
|
|
|
// "test_layer_normalization_2d_axis0_expanded",
|
|
|
|
|
// "test_layer_normalization_2d_axis0",
|
|
|
|
|
// "test_layer_normalization_2d_axis1_expanded",
|
|
|
|
|
// "test_layer_normalization_2d_axis1",
|
|
|
|
|
// // "test_layer_normalization_3d_axis_negative_1_epsilon_expanded",
|
|
|
|
|
// "test_layer_normalization_3d_axis_negative_1_epsilon",
|
|
|
|
|
// // "test_layer_normalization_3d_axis_negative_2_epsilon_expanded",
|
|
|
|
|
// "test_layer_normalization_3d_axis_negative_2_epsilon",
|
|
|
|
|
// // "test_layer_normalization_3d_axis_negative_3_epsilon_expanded",
|
|
|
|
|
// "test_layer_normalization_3d_axis_negative_3_epsilon",
|
|
|
|
|
// // "test_layer_normalization_3d_axis0_epsilon_expanded",
|
|
|
|
|
// "test_layer_normalization_3d_axis0_epsilon",
|
|
|
|
|
// "test_layer_normalization_3d_axis1_epsilon_expanded",
|
|
|
|
|
// "test_layer_normalization_3d_axis1_epsilon",
|
|
|
|
|
// // "test_layer_normalization_3d_axis2_epsilon_expanded",
|
|
|
|
|
// "test_layer_normalization_3d_axis2_epsilon",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_1_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_1",
|
|
|
|
|
// // "test_layer_normalization_4d_axis_negative_2_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_2",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_3_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_3",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_4_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis_negative_4",
|
|
|
|
|
// "test_layer_normalization_4d_axis0_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis0",
|
|
|
|
|
// "test_layer_normalization_4d_axis1_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis1",
|
|
|
|
|
// "test_layer_normalization_4d_axis2_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis2",
|
|
|
|
|
// "test_layer_normalization_4d_axis3_expanded",
|
|
|
|
|
// "test_layer_normalization_4d_axis3",
|
|
|
|
|
// "test_layer_normalization_default_axis_expanded",
|
|
|
|
|
// "test_layer_normalization_default_axis",
|
|
|
|
|
"test_leakyrelu_default",
|
|
|
|
|
"test_leakyrelu_example",
|
|
|
|
|
"test_leakyrelu",
|
|
|
|
|
// "test_less_bcast",
|
|
|
|
|
// "test_less_equal_bcast_expanded",
|
|
|
|
|
// "test_less_equal_bcast",
|
|
|
|
|
// "test_less_equal_expanded",
|
|
|
|
|
// "test_less_equal",
|
|
|
|
|
// "test_less",
|
|
|
|
|
"test_log_example",
|
|
|
|
|
"test_log",
|
|
|
|
|
// // "test_logsoftmax_axis_0_expanded",
|
|
|
|
|
// // "test_logsoftmax_axis_0",
|
|
|
|
|
// // "test_logsoftmax_axis_1_expanded",
|
|
|
|
|
// // "test_logsoftmax_axis_1",
|
|
|
|
|
// // "test_logsoftmax_axis_2_expanded",
|
|
|
|
|
// // "test_logsoftmax_axis_2",
|
|
|
|
|
// // "test_logsoftmax_default_axis_expanded",
|
|
|
|
|
// // "test_logsoftmax_default_axis",
|
|
|
|
|
// // "test_logsoftmax_example_1_expanded",
|
|
|
|
|
// // "test_logsoftmax_example_1",
|
|
|
|
|
// // "test_logsoftmax_large_number_expanded",
|
|
|
|
|
// // "test_logsoftmax_large_number",
|
|
|
|
|
// // "test_logsoftmax_negative_axis_expanded",
|
|
|
|
|
// // "test_logsoftmax_negative_axis",
|
|
|
|
|
// "test_lrn_default",
|
|
|
|
|
// "test_lrn",
|
|
|
|
|
// // "test_lstm_batchwise",
|
|
|
|
|
// // "test_lstm_defaults",
|
|
|
|
|
// // "test_lstm_with_initial_bias",
|
|
|
|
|
// // "test_lstm_with_peepholes",
|
|
|
|
|
// "test_matmul_2d",
|
|
|
|
|
// "test_matmul_3d",
|
|
|
|
|
// "test_matmul_4d",
|
|
|
|
|
// // "test_matmulinteger",
|
|
|
|
|
// "test_max_example",
|
|
|
|
|
// "test_max_float16",
|
|
|
|
|
// "test_max_float32",
|
|
|
|
|
// "test_max_float64",
|
|
|
|
|
// "test_max_int16",
|
|
|
|
|
// "test_max_int32",
|
|
|
|
|
// "test_max_int64",
|
|
|
|
|
// "test_max_int8",
|
|
|
|
|
// "test_max_one_input",
|
|
|
|
|
// "test_max_two_inputs",
|
|
|
|
|
// "test_max_uint16",
|
|
|
|
|
// "test_max_uint32",
|
|
|
|
|
// "test_max_uint64",
|
|
|
|
|
// "test_max_uint8",
|
|
|
|
|
// "test_maxpool_1d_default",
|
|
|
|
|
// "test_maxpool_2d_ceil",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_maxpool_2d_default",
|
2024-01-09 18:10:57 +00:00
|
|
|
// "test_maxpool_2d_dilations",
|
2021-04-27 07:04:25 +00:00
|
|
|
"test_maxpool_2d_pads",
|
|
|
|
|
"test_maxpool_2d_precomputed_pads",
|
|
|
|
|
"test_maxpool_2d_precomputed_same_upper",
|
|
|
|
|
"test_maxpool_2d_precomputed_strides",
|
|
|
|
|
"test_maxpool_2d_same_lower",
|
|
|
|
|
"test_maxpool_2d_same_upper",
|
|
|
|
|
"test_maxpool_2d_strides",
|
2024-01-09 18:10:57 +00:00
|
|
|
// "test_maxpool_2d_uint8",
|
|
|
|
|
// "test_maxpool_3d_default",
|
|
|
|
|
// "test_maxpool_with_argmax_2d_precomputed_pads",
|
|
|
|
|
// "test_maxpool_with_argmax_2d_precomputed_strides",
|
|
|
|
|
// // "test_maxunpool_export_with_output_shape",
|
|
|
|
|
// // "test_maxunpool_export_without_output_shape",
|
|
|
|
|
// // "test_mean_example",
|
|
|
|
|
// // "test_mean_one_input",
|
|
|
|
|
// // "test_mean_two_inputs",
|
|
|
|
|
// // "test_melweightmatrix",
|
|
|
|
|
// "test_min_example",
|
|
|
|
|
// "test_min_float16",
|
|
|
|
|
// "test_min_float32",
|
|
|
|
|
// "test_min_float64",
|
|
|
|
|
// "test_min_int16",
|
|
|
|
|
// "test_min_int32",
|
|
|
|
|
// "test_min_int64",
|
|
|
|
|
// "test_min_int8",
|
|
|
|
|
// "test_min_one_input",
|
|
|
|
|
// "test_min_two_inputs",
|
|
|
|
|
// "test_min_uint16",
|
|
|
|
|
// "test_min_uint32",
|
|
|
|
|
// "test_min_uint64",
|
|
|
|
|
// "test_min_uint8",
|
|
|
|
|
// "test_mod_bcast",
|
|
|
|
|
// "test_mod_broadcast",
|
|
|
|
|
// "test_mod_float_mixed_sign_example",
|
|
|
|
|
// "test_mod_fmod_mixed_sign_example",
|
|
|
|
|
// "test_mod_int64_fmod",
|
|
|
|
|
// "test_mod_int64_mixed_sign_example",
|
|
|
|
|
// "test_mod_mixed_sign_float16",
|
|
|
|
|
// "test_mod_mixed_sign_float32",
|
|
|
|
|
// "test_mod_mixed_sign_float64",
|
|
|
|
|
// "test_mod_mixed_sign_int16",
|
|
|
|
|
// "test_mod_mixed_sign_int32",
|
|
|
|
|
// "test_mod_mixed_sign_int64",
|
|
|
|
|
// "test_mod_mixed_sign_int8",
|
|
|
|
|
// "test_mod_uint16",
|
|
|
|
|
// "test_mod_uint32",
|
|
|
|
|
// "test_mod_uint64",
|
|
|
|
|
// "test_mod_uint8",
|
|
|
|
|
// // "test_momentum_multiple",
|
|
|
|
|
// // "test_momentum",
|
|
|
|
|
"test_mul_bcast",
|
|
|
|
|
"test_mul_example",
|
|
|
|
|
// "test_mul_uint8",
|
|
|
|
|
"test_mul",
|
|
|
|
|
// "test_mvn_expanded",
|
|
|
|
|
// "test_mvn",
|
|
|
|
|
"test_neg_example",
|
|
|
|
|
"test_neg",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_iinput_shape_is_NCd1_weight_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NC_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NC",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_mean_weight_negative_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1_weight",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_no_weight_reduction_mean_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_mean",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_reduction_sum",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_mean",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight_reduction_sum",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2_with_weight",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// // "test_negative_log_likelihood_loss_input_shape_is_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// // "test_nesterov_momentum",
|
|
|
|
|
// // "test_nllloss_NC_expanded",
|
|
|
|
|
// // "test_nllloss_NC",
|
|
|
|
|
// // "test_nllloss_NCd1_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_ii",
|
|
|
|
|
// // "test_nllloss_NCd1_mean_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_mean_weight_negative_ii",
|
|
|
|
|
// // "test_nllloss_NCd1_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_weight_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1_weight_ii",
|
|
|
|
|
// // "test_nllloss_NCd1_weight",
|
|
|
|
|
// // "test_nllloss_NCd1",
|
|
|
|
|
// // "test_nllloss_NCd1d2_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_no_weight_reduction_mean_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_mean_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_mean",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_sum_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_reduction_sum",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_mean",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight_reduction_sum",
|
|
|
|
|
// // "test_nllloss_NCd1d2_with_weight",
|
|
|
|
|
// // "test_nllloss_NCd1d2",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_none_no_weight_negative_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3_sum_weight_high_ii",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// // "test_nllloss_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// "test_nonmaxsuppression_center_point_box_format",
|
|
|
|
|
// "test_nonmaxsuppression_flipped_coordinates",
|
|
|
|
|
// "test_nonmaxsuppression_identical_boxes",
|
|
|
|
|
// "test_nonmaxsuppression_limit_output_size",
|
|
|
|
|
// "test_nonmaxsuppression_single_box",
|
|
|
|
|
// "test_nonmaxsuppression_suppress_by_IOU_and_scores",
|
|
|
|
|
// "test_nonmaxsuppression_suppress_by_IOU",
|
|
|
|
|
// "test_nonmaxsuppression_two_batches",
|
|
|
|
|
// "test_nonmaxsuppression_two_classes",
|
|
|
|
|
// "test_nonzero_example",
|
|
|
|
|
// "test_not_2d",
|
|
|
|
|
// "test_not_3d",
|
|
|
|
|
// "test_not_4d",
|
|
|
|
|
// // "test_onehot_negative_indices",
|
|
|
|
|
// // "test_onehot_with_axis",
|
|
|
|
|
// // "test_onehot_with_negative_axis",
|
|
|
|
|
// // "test_onehot_without_axis",
|
|
|
|
|
// // "test_optional_get_element_sequence",
|
|
|
|
|
// // "test_optional_get_element",
|
|
|
|
|
// // "test_optional_has_element_empty",
|
|
|
|
|
// // "test_optional_has_element",
|
|
|
|
|
// "test_or_bcast3v1d",
|
|
|
|
|
// "test_or_bcast3v2d",
|
|
|
|
|
// "test_or_bcast4v2d",
|
|
|
|
|
// "test_or_bcast4v3d",
|
|
|
|
|
// "test_or_bcast4v4d",
|
|
|
|
|
// "test_or2d",
|
|
|
|
|
// "test_or3d",
|
|
|
|
|
// "test_or4d",
|
|
|
|
|
"test_pow_bcast_array",
|
|
|
|
|
"test_pow_bcast_scalar",
|
|
|
|
|
"test_pow_example",
|
|
|
|
|
// "test_pow_types_float",
|
|
|
|
|
// "test_pow_types_float32_int32",
|
|
|
|
|
// "test_pow_types_float32_int64",
|
|
|
|
|
// "test_pow_types_float32_uint32",
|
|
|
|
|
// "test_pow_types_float32_uint64",
|
|
|
|
|
// "test_pow_types_int",
|
|
|
|
|
// "test_pow_types_int32_float32",
|
|
|
|
|
// "test_pow_types_int32_int32",
|
|
|
|
|
// "test_pow_types_int64_float32",
|
|
|
|
|
// "test_pow_types_int64_int64",
|
|
|
|
|
"test_pow",
|
|
|
|
|
// "test_prelu_broadcast",
|
|
|
|
|
// "test_prelu_example",
|
|
|
|
|
// // "test_qlinearconv",
|
|
|
|
|
// // "test_qlinearmatmul_2D",
|
|
|
|
|
// // "test_qlinearmatmul_3D",
|
|
|
|
|
// // "test_quantizelinear_axis",
|
|
|
|
|
// // "test_quantizelinear",
|
|
|
|
|
// "test_range_float_type_positive_delta_expanded",
|
|
|
|
|
// "test_range_float_type_positive_delta",
|
|
|
|
|
// "test_range_int32_type_negative_delta_expanded",
|
|
|
|
|
// "test_range_int32_type_negative_delta",
|
|
|
|
|
// "test_reciprocal_example",
|
|
|
|
|
// "test_reciprocal",
|
|
|
|
|
// "test_reduce_l1_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_l1_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_l1_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_l1_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_l1_keep_dims_example",
|
|
|
|
|
// "test_reduce_l1_keep_dims_random",
|
|
|
|
|
// "test_reduce_l1_negative_axes_keep_dims_example",
|
|
|
|
|
// "test_reduce_l1_negative_axes_keep_dims_random",
|
|
|
|
|
// "test_reduce_l2_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_l2_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_l2_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_l2_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_l2_keep_dims_example",
|
|
|
|
|
// "test_reduce_l2_keep_dims_random",
|
|
|
|
|
// "test_reduce_l2_negative_axes_keep_dims_example",
|
|
|
|
|
// "test_reduce_l2_negative_axes_keep_dims_random",
|
|
|
|
|
// "test_reduce_log_sum_asc_axes",
|
|
|
|
|
// "test_reduce_log_sum_default",
|
|
|
|
|
// "test_reduce_log_sum_desc_axes",
|
|
|
|
|
// tests "test_reduce_log_sum_exp_*" on opset17/opset18 are excluded because they use float64.
|
|
|
|
|
// "opset{7,8,9}/test_reduce_log_sum_exp_default_axes_keepdims_example",
|
|
|
|
|
// "opset{7,8,9}/test_reduce_log_sum_exp_default_axes_keepdims_random",
|
|
|
|
|
// "opset{7,8,9}/test_reduce_log_sum_exp_do_not_keepdims_example",
|
|
|
|
|
// "opset{7,8,9}/test_reduce_log_sum_exp_do_not_keepdims_random",
|
|
|
|
|
// "opset{7,8,9}/test_reduce_log_sum_exp_keepdims_example",
|
|
|
|
|
// "opset{7,8,9}/test_reduce_log_sum_exp_keepdims_random",
|
|
|
|
|
// "opset11/test_reduce_log_sum_exp_negative_axes_keepdims_example",
|
|
|
|
|
// "opset11/test_reduce_log_sum_exp_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_log_sum_negative_axes",
|
|
|
|
|
// "test_reduce_log_sum",
|
|
|
|
|
// "test_reduce_max_default_axes_keepdim_example",
|
|
|
|
|
// "test_reduce_max_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_max_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_max_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_max_keepdims_example",
|
|
|
|
|
// "test_reduce_max_keepdims_random",
|
|
|
|
|
// "test_reduce_max_negative_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_max_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_mean_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_mean_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_mean_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_mean_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_mean_keepdims_example",
|
|
|
|
|
// "test_reduce_mean_keepdims_random",
|
|
|
|
|
// "test_reduce_mean_negative_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_mean_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_min_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_min_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_min_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_min_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_min_keepdims_example",
|
|
|
|
|
// "test_reduce_min_keepdims_random",
|
|
|
|
|
// "test_reduce_min_negative_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_min_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_prod_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_prod_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_prod_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_prod_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_prod_keepdims_example",
|
|
|
|
|
// "test_reduce_prod_keepdims_random",
|
|
|
|
|
// "test_reduce_prod_negative_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_prod_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_empty_axes_input_noop_example",
|
|
|
|
|
// "test_reduce_sum_empty_axes_input_noop_random",
|
|
|
|
|
// "test_reduce_sum_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_negative_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_square_default_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_square_default_axes_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_square_do_not_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_square_do_not_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_square_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_square_keepdims_random",
|
|
|
|
|
// "test_reduce_sum_square_negative_axes_keepdims_example",
|
|
|
|
|
// "test_reduce_sum_square_negative_axes_keepdims_random",
|
|
|
|
|
// "test_reflect_pad",
|
|
|
|
|
"test_relu",
|
|
|
|
|
// "test_reshape_allowzero_reordered",
|
|
|
|
|
// "test_reshape_extended_dims",
|
|
|
|
|
// "test_reshape_negative_dim",
|
|
|
|
|
// "test_reshape_negative_extended_dims",
|
|
|
|
|
// "test_reshape_one_dim",
|
|
|
|
|
// "test_reshape_reduced_dims",
|
|
|
|
|
// "test_reshape_reordered_all_dims",
|
|
|
|
|
// "test_reshape_reordered_dims",
|
|
|
|
|
// "test_reshape_reordered_last_dims",
|
|
|
|
|
// "test_reshape_zero_and_negative_dim",
|
|
|
|
|
// "test_reshape_zero_dim",
|
|
|
|
|
// "test_resize_downsample_linear",
|
|
|
|
|
// "test_resize_downsample_nearest",
|
|
|
|
|
// "test_resize_downsample_scales_cubic_A_n0p5_exclude_outside",
|
|
|
|
|
// "test_resize_downsample_scales_cubic_align_corners",
|
|
|
|
|
// "test_resize_downsample_scales_cubic",
|
|
|
|
|
// "test_resize_downsample_scales_linear_align_corners",
|
|
|
|
|
// "test_resize_downsample_scales_linear",
|
|
|
|
|
// "test_resize_downsample_scales_nearest",
|
|
|
|
|
// "test_resize_downsample_sizes_cubic",
|
|
|
|
|
// "test_resize_downsample_sizes_linear_pytorch_half_pixel",
|
|
|
|
|
// "test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn",
|
|
|
|
|
// "test_resize_downsample_sizes_nearest",
|
|
|
|
|
// "test_resize_nearest",
|
|
|
|
|
// "test_resize_tf_crop_and_resize",
|
|
|
|
|
// "test_resize_upsample_linear",
|
|
|
|
|
// "test_resize_upsample_nearest",
|
|
|
|
|
// "test_resize_upsample_scales_cubic_A_n0p5_exclude_outside",
|
|
|
|
|
// "test_resize_upsample_scales_cubic_align_corners",
|
|
|
|
|
// "test_resize_upsample_scales_cubic_asymmetric",
|
|
|
|
|
// "test_resize_upsample_scales_cubic",
|
|
|
|
|
// "test_resize_upsample_scales_linear_align_corners",
|
|
|
|
|
// "test_resize_upsample_scales_linear",
|
|
|
|
|
// "test_resize_upsample_scales_nearest",
|
|
|
|
|
// "test_resize_upsample_sizes_cubic",
|
|
|
|
|
// "opset{12,13,17,18}/test_resize_upsample_sizes_nearest_ceil_half_pixel",
|
|
|
|
|
// "opset{12,13,17,18}/test_resize_upsample_sizes_nearest_floor_align_corners",
|
|
|
|
|
// "opset{12,13,17,18}/test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric",
|
|
|
|
|
// "test_resize_upsample_sizes_nearest",
|
|
|
|
|
// // "test_reversesequence_batch",
|
|
|
|
|
// // "test_reversesequence_time",
|
|
|
|
|
// // "test_rnn_seq_length",
|
|
|
|
|
// // "test_roialign_aligned_false",
|
|
|
|
|
// // "test_roialign_aligned_true",
|
|
|
|
|
// // "test_roialign",
|
|
|
|
|
// // "test_round",
|
|
|
|
|
// // "test_scan_sum",
|
|
|
|
|
// // "test_scan9_sum",
|
|
|
|
|
// // "test_scatter_elements_with_axis",
|
|
|
|
|
// // "test_scatter_elements_with_duplicate_indices",
|
|
|
|
|
// // "test_scatter_elements_with_negative_indices",
|
|
|
|
|
// // "test_scatter_elements_without_axis",
|
|
|
|
|
// // "test_scatter_with_axis",
|
|
|
|
|
// // "test_scatter_without_axis",
|
|
|
|
|
// // "test_scatternd_add",
|
|
|
|
|
// // "test_scatternd_multiply",
|
|
|
|
|
// // "test_scatternd",
|
|
|
|
|
// // "test_sce_mean_3d_expanded",
|
|
|
|
|
// // "test_sce_mean_3d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_3d_log_prob",
|
|
|
|
|
// // "test_sce_mean_3d",
|
|
|
|
|
// // "test_sce_mean_expanded",
|
|
|
|
|
// // "test_sce_mean_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_3d",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_4d",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii_log_prob",
|
|
|
|
|
// // "test_sce_mean_no_weight_ii",
|
|
|
|
|
// // "test_sce_mean_weight_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight_ii_3d",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight_ii_4d",
|
|
|
|
|
// // "test_sce_mean_weight_ii_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_ii_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight_ii",
|
|
|
|
|
// // "test_sce_mean_weight_log_prob_expanded",
|
|
|
|
|
// // "test_sce_mean_weight_log_prob",
|
|
|
|
|
// // "test_sce_mean_weight",
|
|
|
|
|
// // "test_sce_mean",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii_log_prob",
|
|
|
|
|
// // "test_sce_NCd1_mean_weight_negative_ii",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3_none_no_weight_negative_ii",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3_sum_weight_high_ii",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob",
|
|
|
|
|
// // "test_sce_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// // "test_sce_none_expanded",
|
|
|
|
|
// // "test_sce_none_log_prob_expanded",
|
|
|
|
|
// // "test_sce_none_log_prob",
|
|
|
|
|
// // "test_sce_none_weights_expanded",
|
|
|
|
|
// // "test_sce_none_weights_log_prob_expanded",
|
|
|
|
|
// // "test_sce_none_weights_log_prob",
|
|
|
|
|
// // "test_sce_none_weights",
|
|
|
|
|
// // "test_sce_none",
|
|
|
|
|
// // "test_sce_sum_expanded",
|
|
|
|
|
// // "test_sce_sum_log_prob_expanded",
|
|
|
|
|
// // "test_sce_sum_log_prob",
|
|
|
|
|
// // "test_sce_sum",
|
|
|
|
|
// "test_selu_default",
|
|
|
|
|
// "test_selu_example",
|
|
|
|
|
// "test_selu",
|
|
|
|
|
// // "test_sequence_insert_at_back",
|
|
|
|
|
// // "test_sequence_insert_at_front",
|
|
|
|
|
// // "test_sequence_map_add_1_sequence_1_tensor_expanded",
|
|
|
|
|
// // "test_sequence_map_add_1_sequence_1_tensor",
|
|
|
|
|
// // "test_sequence_map_add_2_sequences_expanded",
|
|
|
|
|
// // "test_sequence_map_add_2_sequences",
|
|
|
|
|
// // "test_sequence_map_extract_shapes_expanded",
|
|
|
|
|
// // "test_sequence_map_extract_shapes",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence_1_tensor_expanded",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence_1_tensor",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence_expanded",
|
|
|
|
|
// // "test_sequence_map_identity_1_sequence",
|
|
|
|
|
// // "test_sequence_map_identity_2_sequences_expanded",
|
|
|
|
|
// // "test_sequence_map_identity_2_sequences",
|
|
|
|
|
// "test_shrink_hard",
|
|
|
|
|
// "test_shrink_soft",
|
|
|
|
|
"test_sigmoid_example",
|
|
|
|
|
"test_sigmoid",
|
|
|
|
|
// "test_sign",
|
|
|
|
|
// "test_simple_rnn_batchwise",
|
|
|
|
|
// "test_simple_rnn_defaults",
|
|
|
|
|
// "test_simple_rnn_with_initial_bias",
|
|
|
|
|
"test_sin_example",
|
|
|
|
|
"test_sin",
|
|
|
|
|
// "test_sinh_example",
|
|
|
|
|
// "test_sinh",
|
|
|
|
|
// // "test_size_example",
|
|
|
|
|
// // "test_size",
|
|
|
|
|
// "test_slice_default_axes",
|
|
|
|
|
// "test_slice_default_steps",
|
|
|
|
|
// "test_slice_end_out_of_bounds",
|
|
|
|
|
// "test_slice_neg_steps",
|
|
|
|
|
// "test_slice_neg",
|
|
|
|
|
// "test_slice_negative_axes",
|
|
|
|
|
// "test_slice_start_out_of_bounds",
|
|
|
|
|
// "test_slice",
|
|
|
|
|
// "test_softmax_axis_0_expanded",
|
|
|
|
|
"test_softmax_axis_0",
|
|
|
|
|
// "test_softmax_axis_1_expanded",
|
|
|
|
|
"test_softmax_axis_1",
|
|
|
|
|
// "test_softmax_axis_2_expanded",
|
|
|
|
|
"test_softmax_axis_2",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1_mean_weight_negative_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_none_no_weight_negative_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3_sum_weight_high_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_mean_weight",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_input_shape_is_NCd1d2d3d4d5_none_no_weight",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_3d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_3d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_4d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_no_weight_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_3d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_4d",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_ignore_index",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_mean_weight",
|
|
|
|
|
// "test_softmax_cross_entropy_mean",
|
|
|
|
|
// "test_softmax_cross_entropy_none_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_none_weights",
|
|
|
|
|
// "test_softmax_cross_entropy_none",
|
|
|
|
|
// "test_softmax_cross_entropy_sum_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_sum_log_prob_expanded",
|
|
|
|
|
// "test_softmax_cross_entropy_sum_log_prob",
|
|
|
|
|
// "test_softmax_cross_entropy_sum",
|
|
|
|
|
// "opset13/test_softmax_default_axis_expanded",
|
|
|
|
|
"opset13/test_softmax_default_axis",
|
|
|
|
|
// "test_softmax_example_expanded",
|
|
|
|
|
"test_softmax_example",
|
|
|
|
|
// "test_softmax_large_number_expanded",
|
|
|
|
|
"test_softmax_large_number",
|
|
|
|
|
// "test_softmax_negative_axis_expanded",
|
|
|
|
|
"test_softmax_negative_axis",
|
|
|
|
|
// // "test_softplus_example",
|
|
|
|
|
// // "test_softplus",
|
|
|
|
|
// // "test_softsign_example",
|
|
|
|
|
// // "test_softsign",
|
|
|
|
|
// "test_spacetodepth_example",
|
|
|
|
|
// "test_spacetodepth",
|
|
|
|
|
// "test_split_equal_parts_1d",
|
|
|
|
|
// "test_split_equal_parts_2d",
|
|
|
|
|
// "test_split_equal_parts_default_axis",
|
|
|
|
|
// "test_split_variable_parts_1d",
|
|
|
|
|
// "test_split_variable_parts_2d",
|
|
|
|
|
// "test_split_variable_parts_default_axis",
|
|
|
|
|
// "test_split_zero_size_splits",
|
|
|
|
|
"test_sqrt_example",
|
|
|
|
|
"test_sqrt",
|
|
|
|
|
// "test_squeeze_negative_axes",
|
|
|
|
|
// "test_squeeze",
|
|
|
|
|
// // "test_stft_with_window",
|
|
|
|
|
// // "test_stft",
|
|
|
|
|
// // "test_strnormalizer_export_monday_casesensintive_lower",
|
|
|
|
|
// // "test_strnormalizer_export_monday_casesensintive_nochangecase",
|
|
|
|
|
// // "test_strnormalizer_export_monday_casesensintive_upper",
|
|
|
|
|
// // "test_strnormalizer_export_monday_empty_output",
|
|
|
|
|
// // "test_strnormalizer_export_monday_insensintive_upper_twodim",
|
|
|
|
|
// // "test_strnormalizer_nostopwords_nochangecase",
|
|
|
|
|
"test_sub_bcast",
|
|
|
|
|
"test_sub_example",
|
|
|
|
|
// "test_sub_uint8",
|
|
|
|
|
"test_sub",
|
|
|
|
|
// "test_sum_example",
|
|
|
|
|
// "test_sum_one_input",
|
|
|
|
|
// "test_sum_two_inputs",
|
|
|
|
|
"test_tan_example",
|
|
|
|
|
"test_tan",
|
|
|
|
|
"test_tanh_example",
|
|
|
|
|
"test_tanh",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_batch_onlybigrams_skip0",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_batch_onlybigrams_skip5",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_batch_uniandbigrams_skip5",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_only_bigrams_skip0",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_onlybigrams_levelempty",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_onlybigrams_skip5",
|
|
|
|
|
// // "test_tfidfvectorizer_tf_uniandbigrams_skip5",
|
|
|
|
|
// "test_thresholdedrelu_default",
|
|
|
|
|
// "test_thresholdedrelu_example",
|
|
|
|
|
// "test_thresholdedrelu",
|
|
|
|
|
// "test_tile_precomputed",
|
|
|
|
|
// "test_tile",
|
|
|
|
|
// // "test_top_k_negative_axis",
|
|
|
|
|
// // "test_top_k_smallest",
|
|
|
|
|
// // "test_top_k",
|
|
|
|
|
// // "test_training_dropout_default_mask",
|
|
|
|
|
// // "test_training_dropout_default",
|
|
|
|
|
// // "test_training_dropout_mask",
|
|
|
|
|
// // "test_training_dropout_zero_ratio_mask",
|
|
|
|
|
// // "test_training_dropout_zero_ratio",
|
|
|
|
|
// // "test_training_dropout",
|
|
|
|
|
"test_transpose_all_permutations_0",
|
|
|
|
|
"test_transpose_all_permutations_1",
|
|
|
|
|
"test_transpose_all_permutations_2",
|
|
|
|
|
"test_transpose_all_permutations_3",
|
|
|
|
|
"test_transpose_all_permutations_4",
|
|
|
|
|
"test_transpose_all_permutations_5",
|
|
|
|
|
"test_transpose_default"
|
|
|
|
|
// "test_tril_neg",
|
|
|
|
|
// "test_tril_one_row_neg",
|
|
|
|
|
// "test_tril_out_neg",
|
|
|
|
|
// "test_tril_out_pos",
|
|
|
|
|
// "test_tril_pos",
|
|
|
|
|
// "test_tril_square_neg",
|
|
|
|
|
// "test_tril_square",
|
|
|
|
|
// "test_tril_zero",
|
|
|
|
|
// "test_tril",
|
|
|
|
|
// "test_triu_neg",
|
|
|
|
|
// "test_triu_one_row",
|
|
|
|
|
// "test_triu_out_neg_out",
|
|
|
|
|
// "test_triu_out_pos",
|
|
|
|
|
// "test_triu_pos",
|
|
|
|
|
// "test_triu_square_neg",
|
|
|
|
|
// "test_triu_square",
|
|
|
|
|
// "test_triu_zero",
|
|
|
|
|
// "test_triu",
|
|
|
|
|
// // "test_unique_not_sorted_without_axis",
|
|
|
|
|
// // "test_unique_sorted_with_axis_3d",
|
|
|
|
|
// // "test_unique_sorted_with_axis",
|
|
|
|
|
// // "test_unique_sorted_with_negative_axis",
|
|
|
|
|
// // "test_unique_sorted_without_axis",
|
|
|
|
|
// "test_unsqueeze_axis_0",
|
|
|
|
|
// "test_unsqueeze_axis_1",
|
|
|
|
|
// "test_unsqueeze_axis_2",
|
|
|
|
|
// "test_unsqueeze_axis_3",
|
|
|
|
|
// "test_unsqueeze_negative_axes",
|
|
|
|
|
// "test_unsqueeze_three_axes",
|
|
|
|
|
// "test_unsqueeze_two_axes",
|
|
|
|
|
// "test_unsqueeze_unsorted_axes",
|
|
|
|
|
// "test_unsqueeze",
|
|
|
|
|
// "test_wrap_pad"
|
|
|
|
|
// "test_upsample_nearest",
|
|
|
|
|
// "test_where_example",
|
|
|
|
|
// "test_where_long_example",
|
|
|
|
|
// "test_xor_bcast3v1d",
|
|
|
|
|
// "test_xor_bcast3v2d",
|
|
|
|
|
// "test_xor_bcast4v2d",
|
|
|
|
|
// "test_xor_bcast4v3d",
|
|
|
|
|
// "test_xor_bcast4v4d",
|
|
|
|
|
// "test_xor2d",
|
|
|
|
|
// "test_xor3d",
|
|
|
|
|
// "test_xor4d"
|
2021-04-27 07:04:25 +00:00
|
|
|
],
|
|
|
|
|
"ops": []
|
|
|
|
|
}
|
|
|
|
|
}
|