From d13f3153d7a1e87d81095ef5961887c1086f4b73 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Thu, 6 Jul 2023 08:35:19 -0700 Subject: [PATCH] [js/webgpu] enable op test for webgpu (#16542) ### Description This change enables the JSON-format operator tests for webgpu. Usage: ``` npm test -- op abs.jsonc -b=webgpu ``` --- js/web/script/test-runner-cli.ts | 2 + js/web/test/data/ops/conv.jsonc | 5 +- js/web/test/data/ops/leaky-relu.jsonc | 4 +- js/web/test/data/ops/pad-big.jsonc | 1 + js/web/test/data/ops/pad.jsonc | 7 ++ js/web/test/data/ops/reshape-int32.jsonc | 97 ++++++++++++++++ js/web/test/data/ops/reshape.jsonc | 8 +- js/web/test/data/ops/split.jsonc | 2 + js/web/test/suite-test-list.jsonc | 45 +++++++- js/web/test/test-main.ts | 26 +++-- js/web/test/test-runner.ts | 134 ++++++++++++++++++++++- 11 files changed, 309 insertions(+), 22 deletions(-) create mode 100644 js/web/test/data/ops/reshape-int32.jsonc diff --git a/js/web/script/test-runner-cli.ts b/js/web/script/test-runner-cli.ts index 5776f6b906..49f5de3cdd 100644 --- a/js/web/script/test-runner-cli.ts +++ b/js/web/script/test-runner-cli.ts @@ -57,6 +57,7 @@ async function main() { const DEFAULT_OPSET_VERSIONS = fs.readdirSync(TEST_DATA_MODEL_NODE_ROOT, {withFileTypes: true}) .filter(dir => dir.isDirectory() && dir.name.startsWith('opset')) .map(dir => dir.name.slice(5)); + const MAX_OPSET_VERSION = Math.max(...DEFAULT_OPSET_VERSIONS.map(v => Number.parseInt(v, 10))); const FILE_CACHE_ENABLED = args.fileCache; // whether to enable file cache const FILE_CACHE_MAX_FILE_SIZE = 1 * 1024 * 1024; // The max size of the file that will be put into file cache @@ -378,6 +379,7 @@ async function main() { // field 'verbose' and 'backend' is not set for (const test of tests) { test.backend = backend; + test.opsets = test.opsets || [{domain: '', version: MAX_OPSET_VERSION}]; } npmlog.verbose('TestRunnerCli.Init.Op', 'Finished preparing test data.'); npmlog.verbose('TestRunnerCli.Init.Op', '==============================================================='); diff --git a/js/web/test/data/ops/conv.jsonc b/js/web/test/data/ops/conv.jsonc index 35f7360454..e49cc69dbe 100644 --- a/js/web/test/data/ops/conv.jsonc +++ b/js/web/test/data/ops/conv.jsonc @@ -264,7 +264,10 @@ 20.0, 21.0, 22.0, - 23.0 + 23.0, + 0, + 0, + 0 ], "dims": [1, 3, 3, 3], "type": "float32" diff --git a/js/web/test/data/ops/leaky-relu.jsonc b/js/web/test/data/ops/leaky-relu.jsonc index 6352a7b095..7bfa8c64cb 100644 --- a/js/web/test/data/ops/leaky-relu.jsonc +++ b/js/web/test/data/ops/leaky-relu.jsonc @@ -8,14 +8,14 @@ "name": "T[0]", "inputs": [ { - "data": [-1.0, 0.0, 1.0, 1.0, -2.0], + "data": [-1.0, 0.0, 1.0, -2.0], "dims": [1, 4], "type": "float32" } ], "outputs": [ { - "data": [-0.01, 0.0, 1.0, 1.0], + "data": [-0.01, 0.0, 1.0, -0.02], "dims": [1, 4], "type": "float32" } diff --git a/js/web/test/data/ops/pad-big.jsonc b/js/web/test/data/ops/pad-big.jsonc index 8c03685de0..77ad8f3523 100644 --- a/js/web/test/data/ops/pad-big.jsonc +++ b/js/web/test/data/ops/pad-big.jsonc @@ -2,6 +2,7 @@ { "name": "constant 2D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "reflect", "type": "string" }, { "name": "pads", "data": [0, 0, 1, 1, 0, 0, 1, 1], "type": "ints" } diff --git a/js/web/test/data/ops/pad.jsonc b/js/web/test/data/ops/pad.jsonc index 2d0de76516..8a4e0c7174 100644 --- a/js/web/test/data/ops/pad.jsonc +++ b/js/web/test/data/ops/pad.jsonc @@ -2,6 +2,7 @@ { "name": "constant 2D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "constant", "type": "string" }, { "name": "value", "data": 1.2, "type": "float" }, @@ -80,6 +81,7 @@ { "name": "constant 3D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "constant", "type": "string" }, { "name": "value", "data": 2.3, "type": "float" }, @@ -414,6 +416,7 @@ { "name": "Reflect 1D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "reflect", "type": "string" }, { "name": "pads", "data": [5, 7], "type": "ints" } @@ -457,6 +460,7 @@ { "name": "Reflect 2D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "reflect", "type": "string" }, { "name": "pads", "data": [3, 2, 2, 5], "type": "ints" } @@ -548,6 +552,7 @@ { "name": "Reflect 3D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "reflect", "type": "string" }, { "name": "pads", "data": [1, 2, 2, 2, 3, 1], "type": "ints" } @@ -881,6 +886,7 @@ { "name": "Edge 2D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "edge", "type": "string" }, { "name": "pads", "data": [3, 2, 2, 3], "type": "ints" } @@ -958,6 +964,7 @@ { "name": "Edge 3D", "operator": "Pad", + "opsets": [{"domain": "", "version": 10}], "attributes": [ { "name": "mode", "data": "edge", "type": "string" }, { "name": "pads", "data": [1, 2, 2, 2, 3, 1], "type": "ints" } diff --git a/js/web/test/data/ops/reshape-int32.jsonc b/js/web/test/data/ops/reshape-int32.jsonc new file mode 100644 index 0000000000..3b7b115419 --- /dev/null +++ b/js/web/test/data/ops/reshape-int32.jsonc @@ -0,0 +1,97 @@ +[ + { + "name": "Reshape with '0' and '-1' in the shape tensor input", + "operator": "Reshape", + "attributes": [], + "cases": [ + { + "name": "T[0]", + "inputs": [ + { + "data": [1, 1, 1, 1], + "dims": [2, 2], + "type": "float32" + }, + { + "data": [-1, 0], + "dims": [2], + "type": "int32" + } + ], + "outputs": [ + { + "data": [1, 1, 1, 1], + "dims": [2, 2], + "type": "float32" + } + ] + }, + { + "name": "Scalar to 1D", + "inputs": [ + { + "data": [1], + "dims": [], + "type": "float32" + }, + { + "data": [1], + "dims": [1], + "type": "int32" + } + ], + "outputs": [ + { + "data": [1], + "dims": [1], + "type": "float32" + } + ] + }, + { + "name": "Scalar to 2D", + "inputs": [ + { + "data": [1], + "dims": [], + "type": "float32" + }, + { + "data": [1, 1], + "dims": [2], + "type": "int32" + } + ], + "outputs": [ + { + "data": [1], + "dims": [1, 1], + "type": "float32" + } + ] + }, + { + "name": "Scalar to 2D with -1 in shape hints", + "inputs": [ + { + "data": [1], + "dims": [], + "type": "float32" + }, + { + "data": [-1, 1], + "dims": [2], + "type": "int32" + } + ], + "outputs": [ + { + "data": [1], + "dims": [1, 1], + "type": "float32" + } + ] + } + ] + } +] diff --git a/js/web/test/data/ops/reshape.jsonc b/js/web/test/data/ops/reshape.jsonc index 3b7b115419..a7639040f1 100644 --- a/js/web/test/data/ops/reshape.jsonc +++ b/js/web/test/data/ops/reshape.jsonc @@ -15,7 +15,7 @@ { "data": [-1, 0], "dims": [2], - "type": "int32" + "type": "int64" } ], "outputs": [ @@ -37,7 +37,7 @@ { "data": [1], "dims": [1], - "type": "int32" + "type": "int64" } ], "outputs": [ @@ -59,7 +59,7 @@ { "data": [1, 1], "dims": [2], - "type": "int32" + "type": "int64" } ], "outputs": [ @@ -81,7 +81,7 @@ { "data": [-1, 1], "dims": [2], - "type": "int32" + "type": "int64" } ], "outputs": [ diff --git a/js/web/test/data/ops/split.jsonc b/js/web/test/data/ops/split.jsonc index 2eb9e8f14f..06c96fe8ca 100644 --- a/js/web/test/data/ops/split.jsonc +++ b/js/web/test/data/ops/split.jsonc @@ -2,6 +2,7 @@ { "name": "Split on Axis 0", "operator": "Split", + "opsets": [{"domain": "", "version": 12}], "attributes": [ { "name": "axis", "data": 0, "type": "int" }, { "name": "split", "data": [2, 4], "type": "ints" } @@ -34,6 +35,7 @@ { "name": "Split on Axis 1 - 2D", "operator": "Split", + "opsets": [{"domain": "", "version": 12}], "attributes": [ { "name": "axis", "data": 1, "type": "int" }, { "name": "split", "data": [2, 4], "type": "ints" } diff --git a/js/web/test/suite-test-list.jsonc b/js/web/test/suite-test-list.jsonc index 8b73baba0a..42be0e135a 100644 --- a/js/web/test/suite-test-list.jsonc +++ b/js/web/test/suite-test-list.jsonc @@ -271,7 +271,7 @@ "pad-big.jsonc", "pow.jsonc", "pow-big-number.jsonc", - "reshape.jsonc", + "reshape-int32.jsonc", "softmax.jsonc", "sin.jsonc", "split.jsonc", @@ -1319,7 +1319,48 @@ // "test_xor3d", // "test_xor4d" ], - "ops": [] + "ops": [ + "abs.jsonc", + "acos.jsonc", + "add.jsonc", + //"and.jsonc", + "asin.jsonc", + "ceil.jsonc", + //"concat.jsonc", + "conv.jsonc", + "cos.jsonc", + "div.jsonc", + //"depth-to-space.jsonc", + //"equal.jsonc", + "exp.jsonc", + "floor.jsonc", + "gemm.jsonc", + "global-average-pool.jsonc", + //"greater.jsonc", + //"less.jsonc", + //"log.jsonc", + //"matmul.jsonc", // <--- some tests fail (when input is 3D/4D/5D) + "mul.jsonc", + //"neg.jsonc", + //"not.jsonc", + //"or.jsonc", + "leaky-relu.jsonc", + "reduce-min.jsonc", + "relu.jsonc", + //"pad.jsonc", + //"pad-big.jsonc", + "pow.jsonc", + "pow-big-number.jsonc", + "reshape.jsonc", + //"softmax.jsonc", + "sin.jsonc", + //"split.jsonc", + "sqrt.jsonc", + "sub.jsonc", + "tan.jsonc", + "transpose.jsonc" + //"xor.jsonc" + ] }, "wasm": { "onnx": ["resnet50", "squeezenet", "tiny_yolov2", "emotion_ferplus"], diff --git a/js/web/test/test-main.ts b/js/web/test/test-main.ts index b4099aa126..614dc4e16d 100644 --- a/js/web/test/test-main.ts +++ b/js/web/test/test-main.ts @@ -64,7 +64,7 @@ for (const logConfig of ORT_WEB_TEST_CONFIG.log) { Logger.set(logConfig.category, logConfig.config); } -import {ModelTestContext, OpTestContext, runModelTestSet, runOpTest} from './test-runner'; +import {ModelTestContext, OpTestContext, ProtoOpTestContext, runModelTestSet, runOpTest} from './test-runner'; import {readJsonFile} from './test-shared'; // Unit test @@ -131,22 +131,32 @@ for (const group of ORT_WEB_TEST_CONFIG.op) { describe(`#OpTest# - ${group.name}`, () => { for (const test of group.tests) { const describeTest = shouldSkipTest(test) ? describe.skip : describe; - describeTest(`[${test.backend!}]${test.operator} - ${test.name}`, () => { - let context: OpTestContext; + const backend = test.backend!; + const useProtoOpTest = backend !== 'webgl'; + describeTest(`[${backend}]${test.operator} - ${test.name}`, () => { + let context: ProtoOpTestContext|OpTestContext; before('Initialize Context', async () => { - context = new OpTestContext(test); + context = useProtoOpTest ? new ProtoOpTestContext(test) : new OpTestContext(test); await context.init(); if (ORT_WEB_TEST_CONFIG.profile) { - OpTestContext.profiler.start(); + if (context instanceof ProtoOpTestContext) { + context.session.startProfiling(); + } else { + OpTestContext.profiler.start(); + } } }); - after('Dispose Context', () => { + after('Dispose Context', async () => { if (ORT_WEB_TEST_CONFIG.profile) { - OpTestContext.profiler.stop(); + if (context instanceof ProtoOpTestContext) { + context.session.endProfiling(); + } else { + OpTestContext.profiler.stop(); + } } - context.dispose(); + await context.dispose(); }); for (const testCase of test.cases) { diff --git a/js/web/test/test-runner.ts b/js/web/test/test-runner.ts index e05c643e04..810d795c87 100644 --- a/js/web/test/test-runner.ts +++ b/js/web/test/test-runner.ts @@ -14,6 +14,7 @@ import {Logger, Profiler} from '../lib/onnxjs/instrument'; import {Operator} from '../lib/onnxjs/operators'; import {Tensor} from '../lib/onnxjs/tensor'; import {ProtoUtil} from '../lib/onnxjs/util'; +import {tensorDataTypeStringToEnum} from '../lib/wasm/wasm-common'; import {base64toBuffer, createMockGraph, readFile} from './test-shared'; import {Test} from './test-types'; @@ -535,7 +536,7 @@ function initializeOperator( } /** - * a OpTestContext object contains all states in a OpTest + * a OpTestContext object contains all states in a OpTest. used for webgl backend. */ export class OpTestContext { static profiler = Profiler.create(); @@ -553,7 +554,7 @@ export class OpTestContext { this.opTest.opsets ?? [{domain: '', version: 7}]); } - dispose(): void { + async dispose(): Promise { this.inferenceHandler.dispose(); this.sessionHandler.dispose(); } @@ -565,6 +566,124 @@ export class OpTestContext { } } +/** + * a ProtoOpTestContext uses a protobuf model for operator test. used for ORT based backend. + */ +export class ProtoOpTestContext { + private readonly loadedData: Uint8Array; // model data, inputs, outputs + session: ort.InferenceSession; + readonly backendHint: string; + constructor(test: Test.OperatorTest) { + const opsetImport = test.opsets!.map(opset => onnx.OperatorSetIdProto.create(opset)); + const operator = test.operator; + const attribute = test.attributes!.map(attr => { + const protoAttr = onnx.AttributeProto.create({name: attr.name}); + switch (attr.type) { + case 'float': + protoAttr.type = onnx.AttributeProto.AttributeType.FLOAT; + protoAttr.f = attr.data as number; + break; + case 'int': + protoAttr.type = onnx.AttributeProto.AttributeType.INT; + protoAttr.i = attr.data as number; + break; + case 'string': + protoAttr.type = onnx.AttributeProto.AttributeType.STRING; + protoAttr.s = new TextEncoder().encode(attr.data as string); + break; + case 'floats': + protoAttr.type = onnx.AttributeProto.AttributeType.FLOATS; + protoAttr.floats = attr.data as number[]; + break; + case 'ints': + protoAttr.type = onnx.AttributeProto.AttributeType.INTS; + protoAttr.ints = attr.data as number[]; + break; + case 'strings': + protoAttr.type = onnx.AttributeProto.AttributeType.STRINGS; + protoAttr.strings = (attr.data as string[]).map(s => new TextEncoder().encode(s)); + break; + default: + throw new Error(`Unsupported attribute type: ${attr.type}`); + } + return protoAttr; + }); + + if (test.cases.length === 0) { + throw new Error(`No test cases found for test: ${test.name} [${test.operator}]`); + } + const inputCount = test.cases[0].inputs!.length; + const outputCount = test.cases[0].outputs!.length; + if (test.cases.some( + testCase => testCase.inputs!.length !== inputCount || testCase.outputs!.length !== outputCount)) { + throw new Error( + `Test cases for test: ${test.name} [${test.operator}] must have the same number of inputs and outputs`); + } + + const model = onnx.ModelProto.create(); + model.irVersion = onnx.Version.IR_VERSION; + model.opsetImport = opsetImport; + model.graph = onnx.GraphProto.create(); + + model.graph.node = [onnx.NodeProto.create({ + input: test.cases[0].inputs!.map((_, i) => `input_${i}`), + output: test.cases[0].outputs!.map((_, i) => `output_${i}`), + opType: operator, + name: operator, + attribute + })]; + + model.graph.input = test.cases[0].inputs!.map((input, i) => onnx.ValueInfoProto.create({ + name: `input_${i}`, + type: onnx.TypeProto.create({ + tensorType: onnx.TypeProto.Tensor.create({elemType: tensorDataTypeStringToEnum(input.type)}), + }), + })); + + model.graph.output = test.cases[0].outputs!.map((output, i) => onnx.ValueInfoProto.create({ + name: `output_${i}`, + type: onnx.TypeProto.create({ + tensorType: onnx.TypeProto.Tensor.create({elemType: tensorDataTypeStringToEnum(output.type)}), + }), + })); + + model.graph.name = test.name; + + this.backendHint = test.backend!; + this.loadedData = onnx.ModelProto.encode(model).finish(); + } + async init(): Promise { + this.session = await ort.InferenceSession.create(this.loadedData, {executionProviders: [this.backendHint]}); + } + + async dispose(): Promise { + await this.session.release(); + } +} + +async function runProtoOpTestcase( + session: ort.InferenceSession, testCase: Test.OperatorTestCase, validator: TensorResultValidator): Promise { + const feeds: Record = {}; + testCase.inputs!.forEach((input, i) => { + let data: number[]|BigUint64Array|BigInt64Array = input.data; + if (input.type === 'uint64') { + data = BigUint64Array.from(input.data.map(BigInt)); + } else if (input.type === 'int64') { + data = BigInt64Array.from(input.data.map(BigInt)); + } + feeds[`input_${i}`] = new ort.Tensor(input.type, data, input.dims); + }); + const results = await session.run(feeds); + + const outputs = testCase.outputs!.map(output => new ort.Tensor(output.type, output.data, output.dims)); + const actualOutputNames = Object.getOwnPropertyNames(results); + const expectedOutputNames = outputs.map((_, i) => `output_${i}`); + expect(actualOutputNames.length).to.equal(expectedOutputNames.length); + expect(actualOutputNames).to.have.members(expectedOutputNames); + + const actualOutputs = actualOutputNames.map(name => results[name]); + validator.checkApiTensorResult(actualOutputs, outputs); +} function createTensor(dims: number[], type: Tensor.DataType, data: number[]): Tensor { const tensor = new Tensor(dims, type); @@ -604,7 +723,12 @@ async function runOpTestcase( /** * run a single operator test case. */ -export async function runOpTest(testcase: Test.OperatorTestCase, context: OpTestContext): Promise { - await runOpTestcase( - context.inferenceHandler, context.createOperator(), testcase, new TensorResultValidator(context.backendHint)); +export async function runOpTest( + testcase: Test.OperatorTestCase, context: ProtoOpTestContext|OpTestContext): Promise { + if (context instanceof ProtoOpTestContext) { + await runProtoOpTestcase(context.session, testcase, new TensorResultValidator(context.backendHint)); + } else { + await runOpTestcase( + context.inferenceHandler, context.createOperator(), testcase, new TensorResultValidator(context.backendHint)); + } }