From a8557b3f0fbf45a2646b38769d5027dbd676c3f3 Mon Sep 17 00:00:00 2001 From: Ashwini Khade Date: Thu, 10 Sep 2020 17:25:32 -0700 Subject: [PATCH] skip tests when model opset > released opset (#5096) * skip tests when model opset > released opset * remove multiple model load * nit fixes * plus some comments --- nodejs/test/test-runner.ts | 48 +++++++++++++++++++++++++++----------- nodejs/test/test-utils.ts | 34 +++++++++++++-------------- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/nodejs/test/test-runner.ts b/nodejs/test/test-runner.ts index 5748a5d378..40e5d41bfa 100644 --- a/nodejs/test/test-runner.ts +++ b/nodejs/test/test-runner.ts @@ -52,11 +52,26 @@ export function run(testDataFolder: string): void { // add cases describe(`${model}`, () => { - let session: InferenceSession; + let session: InferenceSession|null = null; const skipModel = shouldSkipModel(model, ['cpu']); if (!skipModel) { - before(async () => { - session = await InferenceSession.create(modelPath); + before(async function () { + try { + session = await InferenceSession.create(modelPath); + } catch (e) { + // By default ort allows models with opsets from an official onnx release only. If it encounters + // a model with opset > than released opset, ValidateOpsetForDomain throws an error and model load fails. + // Since this is by design such a failure is acceptable in the context of this test. Therefore we simply + // skip this test. Setting env variable ALLOW_RELEASED_ONNX_OPSET_ONLY=0 allows loading a model with + // opset > released onnx opset. + if (process.env.ALLOW_RELEASED_ONNX_OPSET_ONLY !== '0' && e.message.includes("ValidateOpsetForDomain")) { + session = null; + console.log(`Skipping ${model}. To run this test set env variable ALLOW_RELEASED_ONNX_OPSET_ONLY=0`); + this.skip(); + } else { + throw e; + } + } }); } else { console.log(`[test-runner] skipped: ${model}`); @@ -68,19 +83,24 @@ export function run(testDataFolder: string): void { const expectedOutputs = testCase[1]; if (!skipModel && !inputs.some(t => t === undefined) && !expectedOutputs.some(t => t === undefined)) { it(`case${i}`, async () => { - const feeds = {}; - if (inputs.length !== session.inputNames.length) { - throw new RangeError('input length does not match name list'); - } - for (let i = 0; i < inputs.length; i++) { - feeds[session.inputNames[i]] = inputs[i]; - } - const outputs = await session.run(feeds); + if (session !== null) { + const feeds = {}; + if (inputs.length !== session.inputNames.length) { + throw new RangeError('input length does not match name list'); + } + for (let i = 0; i < inputs.length; i++) { + feeds[session.inputNames[i]] = inputs[i]; + } + const outputs = await session.run(feeds); - let j = 0; - for (const name of session.outputNames) { - assertTensorEqual(outputs[name], expectedOutputs[j++]!); + let j = 0; + for (const name of session.outputNames) { + assertTensorEqual(outputs[name], expectedOutputs[j++]!); + } + } else { + throw new TypeError('session is null'); } + }); } } diff --git a/nodejs/test/test-utils.ts b/nodejs/test/test-utils.ts index 1b5ee56966..cfb55aa782 100644 --- a/nodejs/test/test-utils.ts +++ b/nodejs/test/test-utils.ts @@ -20,7 +20,7 @@ export const SQUEEZENET_INPUT0_DATA: number[] = require(path.join(TEST_DATA_ROOT export const SQUEEZENET_OUTPUT0_DATA: number[] = require(path.join(TEST_DATA_ROOT, 'squeezenet.output0.json')); export const BACKEND_TEST_SERIES_FILTERS: {[name: string]: string[]} = - jsonc.readSync(path.join(ORT_ROOT, 'onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc')); + jsonc.readSync(path.join(ORT_ROOT, 'onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc')); export const NUMERIC_TYPE_MAP = new Map Tensor.DataType>([ @@ -105,7 +105,7 @@ export function assertDataEqual(type: Tensor.Type, actual: Tensor.DataType, expe case 'float32': case 'float64': assertFloatEqual( - actual as number[] | Float32Array | Float64Array, expected as number[] | Float32Array | Float64Array); + actual as number[] | Float32Array | Float64Array, expected as number[] | Float32Array | Float64Array); break; case 'uint8': @@ -127,7 +127,7 @@ export function assertDataEqual(type: Tensor.Type, actual: Tensor.DataType, expe } export function assertFloatEqual( - actual: number[]|Float32Array|Float64Array, expected: number[]|Float32Array|Float64Array): void { + actual: number[]|Float32Array|Float64Array, expected: number[]|Float32Array|Float64Array): void { const THRESHOLD_ABSOLUTE_ERROR = 1.0e-4; const THRESHOLD_RELATIVE_ERROR = 1.000001; @@ -181,19 +181,19 @@ export function loadTensorFromFile(pbFile: string): Tensor { return new Tensor('string', tensorProto.stringData.map(i => i.toString()), dims); } else { switch (tensorProto.dataType) { - // FLOAT = 1, - // UINT8 = 2, - // INT8 = 3, - // UINT16 = 4, - // INT16 = 5, - // INT32 = 6, - // INT64 = 7, - // STRING = 8, - // BOOL = 9, - // FLOAT16 = 10, - // DOUBLE = 11, - // UINT32 = 12, - // UINT64 = 13, + // FLOAT = 1, + // UINT8 = 2, + // INT8 = 3, + // UINT16 = 4, + // INT16 = 5, + // INT32 = 6, + // INT64 = 7, + // STRING = 8, + // BOOL = 9, + // FLOAT16 = 10, + // DOUBLE = 11, + // UINT32 = 12, + // UINT64 = 13, case onnx_proto.onnx.TensorProto.DataType.FLOAT: transferredTypedArray = new Float32Array(tensorProto.rawData.byteLength / 4); type = 'float32'; @@ -242,7 +242,7 @@ export function loadTensorFromFile(pbFile: string): Tensor { throw new Error(`not supported tensor type: ${tensorProto.dataType}`); } const transferredTypedArrayRawDataView = - new Uint8Array(transferredTypedArray.buffer, transferredTypedArray.byteOffset, tensorProto.rawData.byteLength); + new Uint8Array(transferredTypedArray.buffer, transferredTypedArray.byteOffset, tensorProto.rawData.byteLength); transferredTypedArrayRawDataView.set(tensorProto.rawData); return new Tensor(type, transferredTypedArray, dims);