From e65f284476a837aa5d38ab9e58e29d2f61044c6e Mon Sep 17 00:00:00 2001 From: Sunghoon <35605090+hanbitmyths@users.noreply.github.com> Date: Tue, 9 Nov 2021 11:58:47 -0800 Subject: [PATCH] [js/web] Support WebGL for ort format models in benchmarks (#9661) * add p50 in test * Support FusedConv in WebGL * resolve comments * add a comment for longToNumber change Co-authored-by: Yulong Wang --- js/package-lock.json | 6 +++--- js/package.json | 3 ++- .../lib/onnxjs/backends/webgl/op-resolve-rules.ts | 1 + .../lib/onnxjs/backends/webgl/ops/fuse-utils.ts | 6 +++--- js/web/lib/onnxjs/graph.ts | 15 ++++++++------- js/web/lib/onnxjs/opset.ts | 2 +- js/web/lib/onnxjs/util.ts | 7 +++++-- 7 files changed, 23 insertions(+), 17 deletions(-) diff --git a/js/package-lock.json b/js/package-lock.json index a384fe02d8..8274737fa0 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -678,9 +678,9 @@ "dev": true }, "caniuse-lite": { - "version": "1.0.30001220", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001220.tgz", - "integrity": "sha512-pjC2T4DIDyGAKTL4dMvGUQaMUHRmhvPpAgNNTa14jaBWHu+bLQgvpFqElxh9L4829Fdx0PlKiMp3wnYldRtECA==", + "version": "1.0.30001275", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001275.tgz", + "integrity": "sha512-ihJVvj8RX0kn9GgP43HKhb5q9s2XQn4nEQhdldEJvZhCsuiB2XOq6fAMYQZaN6FPWfsr2qU0cdL0CSbETwbJAg==", "dev": true }, "chalk": { diff --git a/js/package.json b/js/package.json index 9c47255a3c..572c7e94bd 100644 --- a/js/package.json +++ b/js/package.json @@ -15,5 +15,6 @@ "lint": "eslint . --ext .ts --ext .tsx", "format": "clang-format --glob=\"{common/lib/**/*.ts,node/{lib,script,test}/**/*.ts,node/src/**/*.{cc,h},web/{lib,script,test}/**/*.ts,react_native/{android,example,ios,lib}/**/*.{ts,mm,java}}\" --style=file -i" }, - "license": "MIT" + "license": "MIT", + "dependencies": {} } diff --git a/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts b/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts index e4c2ea34d5..72b743c1fd 100644 --- a/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts +++ b/js/web/lib/onnxjs/backends/webgl/op-resolve-rules.ts @@ -61,6 +61,7 @@ export const WEBGL_OP_RESOLVE_RULES: readonly OpSet.ResolveRule[] = [ ['Exp', '', '6+', unaryOps.exp], ['Flatten', '', '1+', flatten, parseFlattenAttributes], ['Floor', '', '6+', unaryOps.floor], + ['FusedConv', 'com.microsoft', '1+', conv, parseConvAttributes], ['Gather', '', '1+', gather, parseGatherAttributes], ['Gemm', '', '7-10', gemm, parseGemmAttributesV7], ['Gemm', '', '11+', gemm, parseGemmAttributesV11], diff --git a/js/web/lib/onnxjs/backends/webgl/ops/fuse-utils.ts b/js/web/lib/onnxjs/backends/webgl/ops/fuse-utils.ts index 2296212d14..1e015852dd 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/fuse-utils.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/fuse-utils.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import {Attribute} from '../../../attribute'; +import {MAX_CLIP, MIN_CLIP} from '../../../util'; import {GlslValueFunction} from '../glsl-definitions'; import {glslClip, glslRelu, glslSigmoid} from './unary-op'; @@ -36,11 +37,10 @@ export function getActicationSnippet(attributes: InternalActivationAttributes) { } export const parseInternalActivationAttributes = (attributes: Attribute): InternalActivationAttributes => { - const activation = attributes.getString('__internal_activation', ''); + const activation = attributes.getString('activation', ''); if (activation === 'Clip') { - const clipMax = attributes.getFloat('__clip_max', 3.402823e+38); - const clipMin = attributes.getFloat('__clip_min', -3.402823e+38); + const [clipMin, clipMax] = attributes.getFloats('activation_params', [MIN_CLIP, MAX_CLIP]); return {activation, clipMax, clipMin, activationCacheKey: `${activation}:${clipMin},${clipMax}`}; } return {activation, activationCacheKey: activation}; diff --git a/js/web/lib/onnxjs/graph.ts b/js/web/lib/onnxjs/graph.ts index ba012d09f9..488b3316e9 100644 --- a/js/web/lib/onnxjs/graph.ts +++ b/js/web/lib/onnxjs/graph.ts @@ -758,23 +758,24 @@ class GraphImpl implements Graph, Graph.Transformer { if (child.opType === 'Clip') { if (child.inputs.length === 1) { try { - node.attributes.set('__clip_min', 'float', child.attributes.getFloat('min')); - node.attributes.set('__clip_max', 'float', child.attributes.getFloat('max')); + node.attributes.set( + 'activation_params', 'floats', + [child.attributes.getFloat('min'), child.attributes.getFloat('max')]); } catch (e) { - node.attributes.set('__clip_min', 'float', MIN_CLIP); - node.attributes.set('__clip_max', 'float', MAX_CLIP); + node.attributes.set('activation_params', 'floats', [MIN_CLIP, MAX_CLIP]); } } else if ( child.inputs.length >= 3 && this._allData[child.inputs[1]].tensor !== undefined && this._allData[child.inputs[2]].tensor !== undefined) { - node.attributes.set('__clip_min', 'float', this._allData[child.inputs[1]].tensor!.floatData[0]); - node.attributes.set('__clip_max', 'float', this._allData[child.inputs[2]].tensor!.floatData[0]); + node.attributes.set('activation_params', 'floats', [ + this._allData[child.inputs[1]].tensor!.floatData[0], this._allData[child.inputs[2]].tensor!.floatData[0] + ]); } else { // Skip fusion with clip node since clip min and clip max are not coming from initializer continue; } } - node.attributes.set('__internal_activation', 'string', (child.opType)); + node.attributes.set('activation', 'string', (child.opType)); this.deleteNode(next[0]); } } diff --git a/js/web/lib/onnxjs/opset.ts b/js/web/lib/onnxjs/opset.ts index ccb2ad9030..e23a288b4e 100644 --- a/js/web/lib/onnxjs/opset.ts +++ b/js/web/lib/onnxjs/opset.ts @@ -13,7 +13,7 @@ export declare namespace OpSet { /** * Domain of an opset, it can be an empty string(default value, represent for ai.onnx), or 'ai.onnx.ml' */ - type Domain = ''|'ai.onnx.ml'; + type Domain = ''|'ai.onnx.ml'|'com.microsoft'; /** * A resolve rule consists of 4 or 5 items: opType, opSetDomain, versionSelector, operatorImplementation and diff --git a/js/web/lib/onnxjs/util.ts b/js/web/lib/onnxjs/util.ts index 55a4e3988f..73d3ca88fc 100644 --- a/js/web/lib/onnxjs/util.ts +++ b/js/web/lib/onnxjs/util.ts @@ -472,11 +472,14 @@ export class ProtoUtil { } export class LongUtil { - static longToNumber(n: Long|flatbuffers.Long|number) { + // This function is called to get a number from long type of data for attribute, dim, and ir version, + // which values are signed integers. + // To make it more generic, add an optional paramter to convert to a unsigned number. + static longToNumber(n: Long|flatbuffers.Long|number, unsigned?: boolean) { if (Long.isLong(n)) { return n.toNumber(); } else if (n instanceof flatbuffers.Long) { - return Long.fromValue({low: n.low, high: n.high, unsigned: true}).toNumber(); + return Long.fromValue({low: n.low, high: n.high, unsigned: unsigned ?? false}).toNumber(); } return n; }