mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-23 19:32:23 +00:00
[js/webgpu] enable op test for webgpu (#16542)
### Description This change enables the JSON-format operator tests for webgpu. Usage: ``` npm test -- op abs.jsonc -b=webgpu ```
This commit is contained in:
Yulong Wang
GitHub
parent
d906d48ae9
commit
d13f3153d7
11 changed files with 309 additions and 22 deletions
|
|
@ -57,6 +57,7 @@ async function main() {
|
|||
const DEFAULT_OPSET_VERSIONS = fs.readdirSync(TEST_DATA_MODEL_NODE_ROOT, {withFileTypes: true})
|
||||
.filter(dir => dir.isDirectory() && dir.name.startsWith('opset'))
|
||||
.map(dir => dir.name.slice(5));
|
||||
const MAX_OPSET_VERSION = Math.max(...DEFAULT_OPSET_VERSIONS.map(v => Number.parseInt(v, 10)));
|
||||
|
||||
const FILE_CACHE_ENABLED = args.fileCache; // whether to enable file cache
|
||||
const FILE_CACHE_MAX_FILE_SIZE = 1 * 1024 * 1024; // The max size of the file that will be put into file cache
|
||||
|
|
@ -378,6 +379,7 @@ async function main() {
|
|||
// field 'verbose' and 'backend' is not set
|
||||
for (const test of tests) {
|
||||
test.backend = backend;
|
||||
test.opsets = test.opsets || [{domain: '', version: MAX_OPSET_VERSION}];
|
||||
}
|
||||
npmlog.verbose('TestRunnerCli.Init.Op', 'Finished preparing test data.');
|
||||
npmlog.verbose('TestRunnerCli.Init.Op', '===============================================================');
|
||||
|
|
|
|||
|
|
@ -264,7 +264,10 @@
|
|||
20.0,
|
||||
21.0,
|
||||
22.0,
|
||||
23.0
|
||||
23.0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"dims": [1, 3, 3, 3],
|
||||
"type": "float32"
|
||||
|
|
|
|||
|
|
@ -8,14 +8,14 @@
|
|||
"name": "T[0]",
|
||||
"inputs": [
|
||||
{
|
||||
"data": [-1.0, 0.0, 1.0, 1.0, -2.0],
|
||||
"data": [-1.0, 0.0, 1.0, -2.0],
|
||||
"dims": [1, 4],
|
||||
"type": "float32"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"data": [-0.01, 0.0, 1.0, 1.0],
|
||||
"data": [-0.01, 0.0, 1.0, -0.02],
|
||||
"dims": [1, 4],
|
||||
"type": "float32"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
{
|
||||
"name": "constant 2D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "reflect", "type": "string" },
|
||||
{ "name": "pads", "data": [0, 0, 1, 1, 0, 0, 1, 1], "type": "ints" }
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
{
|
||||
"name": "constant 2D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "constant", "type": "string" },
|
||||
{ "name": "value", "data": 1.2, "type": "float" },
|
||||
|
|
@ -80,6 +81,7 @@
|
|||
{
|
||||
"name": "constant 3D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "constant", "type": "string" },
|
||||
{ "name": "value", "data": 2.3, "type": "float" },
|
||||
|
|
@ -414,6 +416,7 @@
|
|||
{
|
||||
"name": "Reflect 1D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "reflect", "type": "string" },
|
||||
{ "name": "pads", "data": [5, 7], "type": "ints" }
|
||||
|
|
@ -457,6 +460,7 @@
|
|||
{
|
||||
"name": "Reflect 2D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "reflect", "type": "string" },
|
||||
{ "name": "pads", "data": [3, 2, 2, 5], "type": "ints" }
|
||||
|
|
@ -548,6 +552,7 @@
|
|||
{
|
||||
"name": "Reflect 3D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "reflect", "type": "string" },
|
||||
{ "name": "pads", "data": [1, 2, 2, 2, 3, 1], "type": "ints" }
|
||||
|
|
@ -881,6 +886,7 @@
|
|||
{
|
||||
"name": "Edge 2D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "edge", "type": "string" },
|
||||
{ "name": "pads", "data": [3, 2, 2, 3], "type": "ints" }
|
||||
|
|
@ -958,6 +964,7 @@
|
|||
{
|
||||
"name": "Edge 3D",
|
||||
"operator": "Pad",
|
||||
"opsets": [{"domain": "", "version": 10}],
|
||||
"attributes": [
|
||||
{ "name": "mode", "data": "edge", "type": "string" },
|
||||
{ "name": "pads", "data": [1, 2, 2, 2, 3, 1], "type": "ints" }
|
||||
|
|
|
|||
97
js/web/test/data/ops/reshape-int32.jsonc
Normal file
97
js/web/test/data/ops/reshape-int32.jsonc
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
[
|
||||
{
|
||||
"name": "Reshape with '0' and '-1' in the shape tensor input",
|
||||
"operator": "Reshape",
|
||||
"attributes": [],
|
||||
"cases": [
|
||||
{
|
||||
"name": "T[0]",
|
||||
"inputs": [
|
||||
{
|
||||
"data": [1, 1, 1, 1],
|
||||
"dims": [2, 2],
|
||||
"type": "float32"
|
||||
},
|
||||
{
|
||||
"data": [-1, 0],
|
||||
"dims": [2],
|
||||
"type": "int32"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"data": [1, 1, 1, 1],
|
||||
"dims": [2, 2],
|
||||
"type": "float32"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Scalar to 1D",
|
||||
"inputs": [
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [],
|
||||
"type": "float32"
|
||||
},
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [1],
|
||||
"type": "int32"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [1],
|
||||
"type": "float32"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Scalar to 2D",
|
||||
"inputs": [
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [],
|
||||
"type": "float32"
|
||||
},
|
||||
{
|
||||
"data": [1, 1],
|
||||
"dims": [2],
|
||||
"type": "int32"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [1, 1],
|
||||
"type": "float32"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Scalar to 2D with -1 in shape hints",
|
||||
"inputs": [
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [],
|
||||
"type": "float32"
|
||||
},
|
||||
{
|
||||
"data": [-1, 1],
|
||||
"dims": [2],
|
||||
"type": "int32"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"data": [1],
|
||||
"dims": [1, 1],
|
||||
"type": "float32"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
{
|
||||
"data": [-1, 0],
|
||||
"dims": [2],
|
||||
"type": "int32"
|
||||
"type": "int64"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
|
|
@ -37,7 +37,7 @@
|
|||
{
|
||||
"data": [1],
|
||||
"dims": [1],
|
||||
"type": "int32"
|
||||
"type": "int64"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
|
|
@ -59,7 +59,7 @@
|
|||
{
|
||||
"data": [1, 1],
|
||||
"dims": [2],
|
||||
"type": "int32"
|
||||
"type": "int64"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
|
|
@ -81,7 +81,7 @@
|
|||
{
|
||||
"data": [-1, 1],
|
||||
"dims": [2],
|
||||
"type": "int32"
|
||||
"type": "int64"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
{
|
||||
"name": "Split on Axis 0",
|
||||
"operator": "Split",
|
||||
"opsets": [{"domain": "", "version": 12}],
|
||||
"attributes": [
|
||||
{ "name": "axis", "data": 0, "type": "int" },
|
||||
{ "name": "split", "data": [2, 4], "type": "ints" }
|
||||
|
|
@ -34,6 +35,7 @@
|
|||
{
|
||||
"name": "Split on Axis 1 - 2D",
|
||||
"operator": "Split",
|
||||
"opsets": [{"domain": "", "version": 12}],
|
||||
"attributes": [
|
||||
{ "name": "axis", "data": 1, "type": "int" },
|
||||
{ "name": "split", "data": [2, 4], "type": "ints" }
|
||||
|
|
|
|||
|
|
@ -271,7 +271,7 @@
|
|||
"pad-big.jsonc",
|
||||
"pow.jsonc",
|
||||
"pow-big-number.jsonc",
|
||||
"reshape.jsonc",
|
||||
"reshape-int32.jsonc",
|
||||
"softmax.jsonc",
|
||||
"sin.jsonc",
|
||||
"split.jsonc",
|
||||
|
|
@ -1319,7 +1319,48 @@
|
|||
// "test_xor3d",
|
||||
// "test_xor4d"
|
||||
],
|
||||
"ops": []
|
||||
"ops": [
|
||||
"abs.jsonc",
|
||||
"acos.jsonc",
|
||||
"add.jsonc",
|
||||
//"and.jsonc",
|
||||
"asin.jsonc",
|
||||
"ceil.jsonc",
|
||||
//"concat.jsonc",
|
||||
"conv.jsonc",
|
||||
"cos.jsonc",
|
||||
"div.jsonc",
|
||||
//"depth-to-space.jsonc",
|
||||
//"equal.jsonc",
|
||||
"exp.jsonc",
|
||||
"floor.jsonc",
|
||||
"gemm.jsonc",
|
||||
"global-average-pool.jsonc",
|
||||
//"greater.jsonc",
|
||||
//"less.jsonc",
|
||||
//"log.jsonc",
|
||||
//"matmul.jsonc", // <--- some tests fail (when input is 3D/4D/5D)
|
||||
"mul.jsonc",
|
||||
//"neg.jsonc",
|
||||
//"not.jsonc",
|
||||
//"or.jsonc",
|
||||
"leaky-relu.jsonc",
|
||||
"reduce-min.jsonc",
|
||||
"relu.jsonc",
|
||||
//"pad.jsonc",
|
||||
//"pad-big.jsonc",
|
||||
"pow.jsonc",
|
||||
"pow-big-number.jsonc",
|
||||
"reshape.jsonc",
|
||||
//"softmax.jsonc",
|
||||
"sin.jsonc",
|
||||
//"split.jsonc",
|
||||
"sqrt.jsonc",
|
||||
"sub.jsonc",
|
||||
"tan.jsonc",
|
||||
"transpose.jsonc"
|
||||
//"xor.jsonc"
|
||||
]
|
||||
},
|
||||
"wasm": {
|
||||
"onnx": ["resnet50", "squeezenet", "tiny_yolov2", "emotion_ferplus"],
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ for (const logConfig of ORT_WEB_TEST_CONFIG.log) {
|
|||
Logger.set(logConfig.category, logConfig.config);
|
||||
}
|
||||
|
||||
import {ModelTestContext, OpTestContext, runModelTestSet, runOpTest} from './test-runner';
|
||||
import {ModelTestContext, OpTestContext, ProtoOpTestContext, runModelTestSet, runOpTest} from './test-runner';
|
||||
import {readJsonFile} from './test-shared';
|
||||
|
||||
// Unit test
|
||||
|
|
@ -131,22 +131,32 @@ for (const group of ORT_WEB_TEST_CONFIG.op) {
|
|||
describe(`#OpTest# - ${group.name}`, () => {
|
||||
for (const test of group.tests) {
|
||||
const describeTest = shouldSkipTest(test) ? describe.skip : describe;
|
||||
describeTest(`[${test.backend!}]${test.operator} - ${test.name}`, () => {
|
||||
let context: OpTestContext;
|
||||
const backend = test.backend!;
|
||||
const useProtoOpTest = backend !== 'webgl';
|
||||
describeTest(`[${backend}]${test.operator} - ${test.name}`, () => {
|
||||
let context: ProtoOpTestContext|OpTestContext;
|
||||
|
||||
before('Initialize Context', async () => {
|
||||
context = new OpTestContext(test);
|
||||
context = useProtoOpTest ? new ProtoOpTestContext(test) : new OpTestContext(test);
|
||||
await context.init();
|
||||
if (ORT_WEB_TEST_CONFIG.profile) {
|
||||
OpTestContext.profiler.start();
|
||||
if (context instanceof ProtoOpTestContext) {
|
||||
context.session.startProfiling();
|
||||
} else {
|
||||
OpTestContext.profiler.start();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
after('Dispose Context', () => {
|
||||
after('Dispose Context', async () => {
|
||||
if (ORT_WEB_TEST_CONFIG.profile) {
|
||||
OpTestContext.profiler.stop();
|
||||
if (context instanceof ProtoOpTestContext) {
|
||||
context.session.endProfiling();
|
||||
} else {
|
||||
OpTestContext.profiler.stop();
|
||||
}
|
||||
}
|
||||
context.dispose();
|
||||
await context.dispose();
|
||||
});
|
||||
|
||||
for (const testCase of test.cases) {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {Logger, Profiler} from '../lib/onnxjs/instrument';
|
|||
import {Operator} from '../lib/onnxjs/operators';
|
||||
import {Tensor} from '../lib/onnxjs/tensor';
|
||||
import {ProtoUtil} from '../lib/onnxjs/util';
|
||||
import {tensorDataTypeStringToEnum} from '../lib/wasm/wasm-common';
|
||||
|
||||
import {base64toBuffer, createMockGraph, readFile} from './test-shared';
|
||||
import {Test} from './test-types';
|
||||
|
|
@ -535,7 +536,7 @@ function initializeOperator(
|
|||
}
|
||||
|
||||
/**
|
||||
* a OpTestContext object contains all states in a OpTest
|
||||
* a OpTestContext object contains all states in a OpTest. used for webgl backend.
|
||||
*/
|
||||
export class OpTestContext {
|
||||
static profiler = Profiler.create();
|
||||
|
|
@ -553,7 +554,7 @@ export class OpTestContext {
|
|||
this.opTest.opsets ?? [{domain: '', version: 7}]);
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
async dispose(): Promise<void> {
|
||||
this.inferenceHandler.dispose();
|
||||
this.sessionHandler.dispose();
|
||||
}
|
||||
|
|
@ -565,6 +566,124 @@ export class OpTestContext {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* a ProtoOpTestContext uses a protobuf model for operator test. used for ORT based backend.
|
||||
*/
|
||||
export class ProtoOpTestContext {
|
||||
private readonly loadedData: Uint8Array; // model data, inputs, outputs
|
||||
session: ort.InferenceSession;
|
||||
readonly backendHint: string;
|
||||
constructor(test: Test.OperatorTest) {
|
||||
const opsetImport = test.opsets!.map(opset => onnx.OperatorSetIdProto.create(opset));
|
||||
const operator = test.operator;
|
||||
const attribute = test.attributes!.map(attr => {
|
||||
const protoAttr = onnx.AttributeProto.create({name: attr.name});
|
||||
switch (attr.type) {
|
||||
case 'float':
|
||||
protoAttr.type = onnx.AttributeProto.AttributeType.FLOAT;
|
||||
protoAttr.f = attr.data as number;
|
||||
break;
|
||||
case 'int':
|
||||
protoAttr.type = onnx.AttributeProto.AttributeType.INT;
|
||||
protoAttr.i = attr.data as number;
|
||||
break;
|
||||
case 'string':
|
||||
protoAttr.type = onnx.AttributeProto.AttributeType.STRING;
|
||||
protoAttr.s = new TextEncoder().encode(attr.data as string);
|
||||
break;
|
||||
case 'floats':
|
||||
protoAttr.type = onnx.AttributeProto.AttributeType.FLOATS;
|
||||
protoAttr.floats = attr.data as number[];
|
||||
break;
|
||||
case 'ints':
|
||||
protoAttr.type = onnx.AttributeProto.AttributeType.INTS;
|
||||
protoAttr.ints = attr.data as number[];
|
||||
break;
|
||||
case 'strings':
|
||||
protoAttr.type = onnx.AttributeProto.AttributeType.STRINGS;
|
||||
protoAttr.strings = (attr.data as string[]).map(s => new TextEncoder().encode(s));
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported attribute type: ${attr.type}`);
|
||||
}
|
||||
return protoAttr;
|
||||
});
|
||||
|
||||
if (test.cases.length === 0) {
|
||||
throw new Error(`No test cases found for test: ${test.name} [${test.operator}]`);
|
||||
}
|
||||
const inputCount = test.cases[0].inputs!.length;
|
||||
const outputCount = test.cases[0].outputs!.length;
|
||||
if (test.cases.some(
|
||||
testCase => testCase.inputs!.length !== inputCount || testCase.outputs!.length !== outputCount)) {
|
||||
throw new Error(
|
||||
`Test cases for test: ${test.name} [${test.operator}] must have the same number of inputs and outputs`);
|
||||
}
|
||||
|
||||
const model = onnx.ModelProto.create();
|
||||
model.irVersion = onnx.Version.IR_VERSION;
|
||||
model.opsetImport = opsetImport;
|
||||
model.graph = onnx.GraphProto.create();
|
||||
|
||||
model.graph.node = [onnx.NodeProto.create({
|
||||
input: test.cases[0].inputs!.map((_, i) => `input_${i}`),
|
||||
output: test.cases[0].outputs!.map((_, i) => `output_${i}`),
|
||||
opType: operator,
|
||||
name: operator,
|
||||
attribute
|
||||
})];
|
||||
|
||||
model.graph.input = test.cases[0].inputs!.map((input, i) => onnx.ValueInfoProto.create({
|
||||
name: `input_${i}`,
|
||||
type: onnx.TypeProto.create({
|
||||
tensorType: onnx.TypeProto.Tensor.create({elemType: tensorDataTypeStringToEnum(input.type)}),
|
||||
}),
|
||||
}));
|
||||
|
||||
model.graph.output = test.cases[0].outputs!.map((output, i) => onnx.ValueInfoProto.create({
|
||||
name: `output_${i}`,
|
||||
type: onnx.TypeProto.create({
|
||||
tensorType: onnx.TypeProto.Tensor.create({elemType: tensorDataTypeStringToEnum(output.type)}),
|
||||
}),
|
||||
}));
|
||||
|
||||
model.graph.name = test.name;
|
||||
|
||||
this.backendHint = test.backend!;
|
||||
this.loadedData = onnx.ModelProto.encode(model).finish();
|
||||
}
|
||||
async init(): Promise<void> {
|
||||
this.session = await ort.InferenceSession.create(this.loadedData, {executionProviders: [this.backendHint]});
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
await this.session.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function runProtoOpTestcase(
|
||||
session: ort.InferenceSession, testCase: Test.OperatorTestCase, validator: TensorResultValidator): Promise<void> {
|
||||
const feeds: Record<string, ort.Tensor> = {};
|
||||
testCase.inputs!.forEach((input, i) => {
|
||||
let data: number[]|BigUint64Array|BigInt64Array = input.data;
|
||||
if (input.type === 'uint64') {
|
||||
data = BigUint64Array.from(input.data.map(BigInt));
|
||||
} else if (input.type === 'int64') {
|
||||
data = BigInt64Array.from(input.data.map(BigInt));
|
||||
}
|
||||
feeds[`input_${i}`] = new ort.Tensor(input.type, data, input.dims);
|
||||
});
|
||||
const results = await session.run(feeds);
|
||||
|
||||
const outputs = testCase.outputs!.map(output => new ort.Tensor(output.type, output.data, output.dims));
|
||||
const actualOutputNames = Object.getOwnPropertyNames(results);
|
||||
const expectedOutputNames = outputs.map((_, i) => `output_${i}`);
|
||||
expect(actualOutputNames.length).to.equal(expectedOutputNames.length);
|
||||
expect(actualOutputNames).to.have.members(expectedOutputNames);
|
||||
|
||||
const actualOutputs = actualOutputNames.map(name => results[name]);
|
||||
validator.checkApiTensorResult(actualOutputs, outputs);
|
||||
}
|
||||
|
||||
function createTensor(dims: number[], type: Tensor.DataType, data: number[]): Tensor {
|
||||
const tensor = new Tensor(dims, type);
|
||||
|
|
@ -604,7 +723,12 @@ async function runOpTestcase(
|
|||
/**
|
||||
* run a single operator test case.
|
||||
*/
|
||||
export async function runOpTest(testcase: Test.OperatorTestCase, context: OpTestContext): Promise<void> {
|
||||
await runOpTestcase(
|
||||
context.inferenceHandler, context.createOperator(), testcase, new TensorResultValidator(context.backendHint));
|
||||
export async function runOpTest(
|
||||
testcase: Test.OperatorTestCase, context: ProtoOpTestContext|OpTestContext): Promise<void> {
|
||||
if (context instanceof ProtoOpTestContext) {
|
||||
await runProtoOpTestcase(context.session, testcase, new TensorResultValidator(context.backendHint));
|
||||
} else {
|
||||
await runOpTestcase(
|
||||
context.inferenceHandler, context.createOperator(), testcase, new TensorResultValidator(context.backendHint));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue