From cbee84ddfb68b50246ce76ce59338526b2f65bde Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:50:11 -0700 Subject: [PATCH] [js/web] allow optional input/output in operator test (#17184) ### Description allow optional input/output in operator test --- js/web/test/data/ops/_example.jsonc | 7 ++ js/web/test/op-test-schema.json | 187 +++++++++++++++++++--------- js/web/test/test-runner.ts | 66 +++++++--- js/web/test/test-types.ts | 12 +- 4 files changed, 190 insertions(+), 82 deletions(-) diff --git a/js/web/test/data/ops/_example.jsonc b/js/web/test/data/ops/_example.jsonc index 1c9f306a4c..79d084a68d 100644 --- a/js/web/test/data/ops/_example.jsonc +++ b/js/web/test/data/ops/_example.jsonc @@ -84,6 +84,13 @@ "dims": [1, 1, 3, 3], "type": "float32" }, + // an input or output can be optional, depending on the operator. + // if an input or output is optional, we can specify it as follows: + // + // { + // "data": null, + // "type": "float32" + // } { "data": [1, 2, 3, 4], "dims": [1, 1, 2, 2], diff --git a/js/web/test/op-test-schema.json b/js/web/test/op-test-schema.json index aa08e29386..d6eab6a4ba 100644 --- a/js/web/test/op-test-schema.json +++ b/js/web/test/op-test-schema.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema", "type": "array", "items": { + "type": "object", "properties": { "name": { "type": "string", @@ -170,78 +171,140 @@ "type": "array", "description": "the test case inputs", "items": { - "properties": { - "type": { - "enum": [ - "float32", - "float64", - "int8", - "int16", - "int32", - "int64", - "uint8", - "uint16", - "uint32", - "uint64", - "bool", - "string" - ] + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "enum": [ + "float32", + "float64", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "bool", + "string" + ] + }, + "data": { + "type": "array", + "items": { + "type": ["number", "string", "boolean"] + } + }, + "dims": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + }, + "required": ["type", "data", "dims"], + "additionalProperties": false }, - "data": { - "type": "array", - "items": { - "type": ["number", "string", "boolean"] - } - }, - "dims": { - "type": "array", - "items": { - "type": "integer", - "minimum": 0 - } + { + "type": "object", + "properties": { + "type": { + "enum": [ + "float32", + "float64", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "bool", + "string" + ] + }, + "data": { + "type": "null" + } + }, + "required": ["type", "data"], + "additionalProperties": false } - }, - "required": ["type", "data", "dims"], - "additionalProperties": false + ] } }, "outputs": { "type": "array", "description": "the test case outputs", "items": { - "properties": { - "type": { - "enum": [ - "float32", - "float64", - "int8", - "int16", - "int32", - "int64", - "uint8", - "uint16", - "uint32", - "uint64", - "bool", - "string" - ] + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "enum": [ + "float32", + "float64", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "bool", + "string" + ] + }, + "data": { + "type": "array", + "items": { + "type": ["number", "string", "boolean"] + } + }, + "dims": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + }, + "required": ["type", "data", "dims"], + "additionalProperties": false }, - "data": { - "type": "array", - "items": { - "type": ["number", "string", "boolean"] - } - }, - "dims": { - "type": "array", - "items": { - "type": "integer", - "minimum": 0 - } + { + "type": "object", + "properties": { + "type": { + "enum": [ + "float32", + "float64", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "bool", + "string" + ] + }, + "data": { + "type": "null" + } + }, + "required": ["type", "data"], + "additionalProperties": false } - }, - "required": ["type", "data", "dims"], - "additionalProperties": false + ] } } }, diff --git a/js/web/test/test-runner.ts b/js/web/test/test-runner.ts index 916243e3d4..17232168a2 100644 --- a/js/web/test/test-runner.ts +++ b/js/web/test/test-runner.ts @@ -641,25 +641,39 @@ export class ProtoOpTestContext { // if inputShapeDefinitions is not specified, use undefined for all inputs normalizedInputShapeDefinitions = new Array(inputCount).fill(undefined); } else if (test.inputShapeDefinitions === 'rankOnly') { + // check if all test cases have data + if (test.cases.some(testCase => testCase.inputs!.some(input => !input.data || !input.dims))) { + throw new Error(`Test cases for test: ${test.name} [${ + test.operator}] must have data for each inputs when inputShapeDefinitions is 'rankOnly'`); + } + // if inputShapeDefinitions is 'rankOnly', use semantic names for all inputs. This means only rank is specified. normalizedInputShapeDefinitions = - test.cases[0].inputs!.map((input, i) => input.dims.map((_, j) => `_input_${i}_d${j}`)); + test.cases[0].inputs!.map((input: Test.TensorValue, i) => input.dims.map((_, j) => `_input_${i}_d${j}`)); // check if all test cases have the same rank for each inputs if (test.cases.some( - testCase => - testCase.inputs!.some((input, i) => input.dims.length !== test.cases[0].inputs![i].dims.length))) { + testCase => testCase.inputs!.some( + (input: Test.TensorValue, i) => + input.dims.length !== (test.cases[0].inputs![i] as Test.TensorValue).dims.length))) { throw new Error(`Test cases for test: ${test.name} [${ test.operator}] must have the same rank for each inputs in different test cases`); } } else if (test.inputShapeDefinitions === 'static') { + // check if all test cases have data + if (test.cases.some(testCase => testCase.inputs!.some(input => !input.data || !input.dims))) { + throw new Error(`Test cases for test: ${test.name} [${ + test.operator}] must have data for each inputs when inputShapeDefinitions is 'rankOnly'`); + } + // if inputShapeDefinitions is 'static', use the shape of the first test case for all inputs. - normalizedInputShapeDefinitions = test.cases[0].inputs!.map(input => input.dims); + normalizedInputShapeDefinitions = test.cases[0].inputs!.map((input: Test.TensorValue) => input.dims); // check if all test cases have the same shape for each inputs if (test.cases.some( testCase => testCase.inputs!.some( - (input, i) => TensorResultValidator.integerEqual(input.dims, test.cases[0].inputs![i].dims)))) { + (input: Test.TensorValue, i) => TensorResultValidator.integerEqual( + input.dims, (test.cases[0].inputs![i] as Test.TensorValue).dims)))) { throw new Error(`Test cases for test: ${test.name} [${ test.operator}] must have the same shape for each inputs in different test cases`); } @@ -727,19 +741,35 @@ async function runProtoOpTestcase( session: ort.InferenceSession, testCase: Test.OperatorTestCase, validator: TensorResultValidator): Promise { const feeds: Record = {}; testCase.inputs!.forEach((input, i) => { - let data: number[]|BigUint64Array|BigInt64Array = input.data; - if (input.type === 'uint64') { - data = BigUint64Array.from(input.data.map(BigInt)); - } else if (input.type === 'int64') { - data = BigInt64Array.from(input.data.map(BigInt)); + if (input.data) { + let data: number[]|BigUint64Array|BigInt64Array = input.data; + if (input.type === 'uint64') { + data = BigUint64Array.from(input.data.map(BigInt)); + } else if (input.type === 'int64') { + data = BigInt64Array.from(input.data.map(BigInt)); + } + feeds[`input_${i}`] = new ort.Tensor(input.type, data, input.dims); } - feeds[`input_${i}`] = new ort.Tensor(input.type, data, input.dims); }); + + const outputs: ort.Tensor[] = []; + const expectedOutputNames: string[] = []; + testCase.outputs!.forEach((output, i) => { + if (output.data) { + let data: number[]|BigUint64Array|BigInt64Array = output.data; + if (output.type === 'uint64') { + data = BigUint64Array.from(output.data.map(BigInt)); + } else if (output.type === 'int64') { + data = BigInt64Array.from(output.data.map(BigInt)); + } + outputs.push(new ort.Tensor(output.type, data, output.dims)); + expectedOutputNames.push(`output_${i}`); + } + }); + const results = await session.run(feeds); - const outputs = testCase.outputs!.map(output => new ort.Tensor(output.type, output.data, output.dims)); const actualOutputNames = Object.getOwnPropertyNames(results); - const expectedOutputNames = outputs.map((_, i) => `output_${i}`); expect(actualOutputNames.length).to.equal(expectedOutputNames.length); expect(actualOutputNames).to.have.members(expectedOutputNames); @@ -758,11 +788,11 @@ function createTensor(dims: number[], type: Tensor.DataType, data: number[]): Te async function runOpTestcase( inferenceHandler: InferenceHandler, operator: Operator, testcase: Test.OperatorTestCase, validator: TensorResultValidator): Promise { - testcase.inputs.forEach((input, i) => { + testcase.inputs.forEach((input: Test.TensorValue, i) => { Logger.verbose('TestOpRunner', ` Input '${i}': ${input.type}[${input.dims.join(',')}]`); }); - const inputTensors = - testcase.inputs.map(input => createTensor(input.dims, input.type as Tensor.DataType, input.data)); + const inputTensors = testcase.inputs.map( + (input: Test.TensorValue) => createTensor(input.dims, input.type as Tensor.DataType, input.data)); const results = operator.impl(inferenceHandler, inputTensors, operator.context); @@ -777,8 +807,8 @@ async function runOpTestcase( results.forEach((output, i) => { Logger.verbose('TestOpRunner', ` Result'${i}': ${output.type}[${output.dims.join(',')}]`); }); - const expectedTensors = - testcase.outputs.map(output => createTensor(output.dims, output.type as Tensor.DataType, output.data)); + const expectedTensors = testcase.outputs.map( + (output: Test.TensorValue) => createTensor(output.dims, output.type as Tensor.DataType, output.data)); validator.checkTensorResult(results, expectedTensors); } diff --git a/js/web/test/test-types.ts b/js/web/test/test-types.ts index b86ac4e50c..db01082b9f 100644 --- a/js/web/test/test-types.ts +++ b/js/web/test/test-types.ts @@ -29,6 +29,14 @@ export declare namespace Test { type: Tensor.Type; } + /** + * This interface represent a placeholder for an empty tensor. Should only be used in testing. + */ + interface EmptyTensorValue { + data: null; + type: Tensor.Type; + } + /** * Represent a string to describe the current environment. * Used in ModelTest and OperatorTest to determine whether to run the test or not. @@ -57,8 +65,8 @@ export declare namespace Test { export interface OperatorTestCase { name: string; - inputs: readonly TensorValue[]; - outputs: readonly TensorValue[]; + inputs: ReadonlyArray; + outputs: ReadonlyArray; } export interface OperatorTestOpsetImport {