mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
f972d21e81
### Description
This PR includes the following changes:
- upgrade js dependencies
- enable STRICT mode for web assembly build.
- corresponding fix for cmake-js upgrade
- corresponsing fix for linter upgrade
- upgrade default typescript compile option of:
- `moduleResolution`: from `node` to `node16`
- `target`: from `es2017` to `es2020`
- fix ESM module import in commonJS source file
## change explanation
### changes to onnxruntime_webassembly.cmake
`-s WASM=1` and `-s LLD_REPORT_UNDEFINED` in latest version is
by-default and deprecated.
### changes to onnxruntime_node.cmake
The npm package `cmake-js` updated its way to find file `node.lib`.
previously it downloads this file from Node.js public release channel,
and now it generates it from a definition file.
The node.js release channel does not contain a windows/arm64 version, so
previously cmake-js will fail to download `node.lib` for that platform.
this is why we made special handling to download the unofficial binary
to build. now this is no longer needed so we removed that from the cmake
file.
### changes to tsconfig.json
`node16` module resolution supports async import and `es2020` as target
supports top level await.
595 lines
25 KiB
TypeScript
595 lines
25 KiB
TypeScript
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
// Licensed under the MIT License.
|
|
|
|
/* eslint-disable guard-for-in */
|
|
/* eslint-disable @typescript-eslint/no-use-before-define */
|
|
|
|
import {spawnSync} from 'child_process';
|
|
import * as fs from 'fs-extra';
|
|
import {default as minimatch} from 'minimatch';
|
|
import npmlog from 'npmlog';
|
|
import * as os from 'os';
|
|
import * as path from 'path';
|
|
import {inspect} from 'util';
|
|
|
|
import {bufferToBase64} from '../test/test-shared';
|
|
import {Test} from '../test/test-types';
|
|
|
|
import {parseTestRunnerCliArgs, TestRunnerCliArgs} from './test-runner-cli-args';
|
|
|
|
async function main() {
|
|
// use dynamic import so that we can use ESM only libraries in commonJS.
|
|
const {globbySync} = await import('globby');
|
|
const stripJsonComments = (await import('strip-json-comments')).default;
|
|
|
|
npmlog.info('TestRunnerCli', 'Initializing...');
|
|
|
|
const args = parseTestRunnerCliArgs(process.argv.slice(2));
|
|
|
|
npmlog.verbose('TestRunnerCli.Init.Config', inspect(args));
|
|
|
|
const TEST_ROOT = path.join(__dirname, '..', 'test');
|
|
const TEST_DATA_MODEL_NODE_ROOT = path.join(TEST_ROOT, 'data', 'node');
|
|
const TEST_DATA_OP_ROOT = path.join(TEST_ROOT, 'data', 'ops');
|
|
|
|
const TEST_DATA_BASE = args.env === 'node' ? TEST_ROOT : '/base/test/';
|
|
|
|
npmlog.verbose('TestRunnerCli.Init', 'Ensure test data folder...');
|
|
fs.ensureSymlinkSync(path.join(__dirname, '../../test/data/node'), TEST_DATA_MODEL_NODE_ROOT, 'junction');
|
|
npmlog.verbose('TestRunnerCli.Init', 'Ensure test data folder... DONE');
|
|
|
|
let testlist: Test.TestList;
|
|
const shouldLoadSuiteTestData = (args.mode === 'suite0' || args.mode === 'suite1');
|
|
if (shouldLoadSuiteTestData) {
|
|
npmlog.verbose('TestRunnerCli.Init', 'Loading testlist...');
|
|
|
|
// The following is a list of unittests for already implemented operators.
|
|
// Modify this list to control what node tests to run.
|
|
const jsonWithComments = fs.readFileSync(path.resolve(TEST_ROOT, './suite-test-list.jsonc')).toString();
|
|
const json = stripJsonComments(jsonWithComments, {whitespace: true});
|
|
testlist = JSON.parse(json) as Test.TestList;
|
|
npmlog.verbose('TestRunnerCli.Init', 'Loading testlist... DONE');
|
|
}
|
|
|
|
// The default backends and opset version lists. Those will be used in suite tests.
|
|
const DEFAULT_BACKENDS: readonly TestRunnerCliArgs.Backend[] =
|
|
args.env === 'node' ? ['cpu', 'wasm'] : ['wasm', 'webgl'];
|
|
const DEFAULT_OPSET_VERSIONS: readonly number[] = [13, 12, 11, 10, 9, 8, 7];
|
|
|
|
const FILE_CACHE_ENABLED = args.fileCache; // whether to enable file cache
|
|
const FILE_CACHE_MAX_FILE_SIZE = 1 * 1024 * 1024; // The max size of the file that will be put into file cache
|
|
const FILE_CACHE_SPLIT_SIZE = 4 * 1024 * 1024; // The min size of the cache file
|
|
const fileCache: Test.FileCache = {};
|
|
|
|
const nodeTests = new Map<string, Test.ModelTestGroup[]>();
|
|
const onnxTests = new Map<string, Test.ModelTestGroup>();
|
|
const opTests = new Map<string, Test.OperatorTestGroup[]>();
|
|
|
|
if (shouldLoadSuiteTestData) {
|
|
npmlog.verbose('TestRunnerCli.Init', 'Loading test groups for suite test...');
|
|
|
|
for (const backend of DEFAULT_BACKENDS) {
|
|
for (const version of DEFAULT_OPSET_VERSIONS) {
|
|
let nodeTest = nodeTests.get(backend);
|
|
if (!nodeTest) {
|
|
nodeTest = [];
|
|
nodeTests.set(backend, nodeTest);
|
|
}
|
|
nodeTest.push(loadNodeTests(backend, version));
|
|
}
|
|
opTests.set(backend, loadOpTests(backend));
|
|
}
|
|
}
|
|
|
|
if (shouldLoadSuiteTestData) {
|
|
npmlog.verbose('TestRunnerCli.Init', 'Loading test groups for suite test... DONE');
|
|
|
|
npmlog.verbose('TestRunnerCli.Init', 'Validate testlist...');
|
|
validateTestList();
|
|
npmlog.verbose('TestRunnerCli.Init', 'Validate testlist... DONE');
|
|
}
|
|
|
|
const modelTestGroups: Test.ModelTestGroup[] = [];
|
|
const opTestGroups: Test.OperatorTestGroup[] = [];
|
|
let unittest = false;
|
|
|
|
npmlog.verbose('TestRunnerCli.Init', 'Preparing test config...');
|
|
switch (args.mode) {
|
|
case 'suite0':
|
|
case 'suite1':
|
|
for (const backend of DEFAULT_BACKENDS) {
|
|
if (args.backends.indexOf(backend) !== -1) {
|
|
modelTestGroups.push(...nodeTests.get(backend)!); // model test : node
|
|
opTestGroups.push(...opTests.get(backend)!); // operator test
|
|
}
|
|
}
|
|
if (args.mode === 'suite0') {
|
|
unittest = true;
|
|
}
|
|
break;
|
|
|
|
case 'model':
|
|
if (!args.param) {
|
|
throw new Error('the test folder should be specified in mode \'node\'');
|
|
} else {
|
|
const testFolderSearchPattern = args.param;
|
|
const testFolder = tryLocateModelTestFolder(testFolderSearchPattern);
|
|
for (const b of args.backends) {
|
|
modelTestGroups.push({name: testFolder, tests: [modelTestFromFolder(testFolder, b, undefined, args.times)]});
|
|
}
|
|
}
|
|
break;
|
|
|
|
case 'unittest':
|
|
unittest = true;
|
|
break;
|
|
|
|
case 'op':
|
|
if (!args.param) {
|
|
throw new Error('the test manifest should be specified in mode \'op\'');
|
|
} else {
|
|
const manifestFileSearchPattern = args.param;
|
|
const manifestFile = tryLocateOpTestManifest(manifestFileSearchPattern);
|
|
for (const b of args.backends) {
|
|
opTestGroups.push(opTestFromManifest(manifestFile, b));
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
throw new Error(`unsupported mode '${args.mode}'`);
|
|
}
|
|
|
|
npmlog.verbose('TestRunnerCli.Init', 'Preparing test config... DONE');
|
|
|
|
npmlog.info('TestRunnerCli', 'Initialization completed. Start to run tests...');
|
|
run({
|
|
unittest,
|
|
model: modelTestGroups,
|
|
op: opTestGroups,
|
|
log: args.logConfig,
|
|
profile: args.profile,
|
|
options: {
|
|
sessionOptions:
|
|
{graphOptimizationLevel: args.graphOptimizationLevel, optimizedModelFilePath: args.optimizedModelFilePath},
|
|
debug: args.debug,
|
|
cpuOptions: args.cpuOptions,
|
|
webglOptions: args.webglOptions,
|
|
wasmOptions: args.wasmOptions,
|
|
globalEnvFlags: args.globalEnvFlags
|
|
}
|
|
});
|
|
npmlog.info('TestRunnerCli', 'Tests completed successfully');
|
|
|
|
function validateTestList() {
|
|
for (const backend of DEFAULT_BACKENDS) {
|
|
const nodeTest = nodeTests.get(backend);
|
|
if (nodeTest) {
|
|
for (const testCase of testlist[backend].node) {
|
|
const testCaseName = typeof testCase === 'string' ? testCase : testCase.name;
|
|
let found = false;
|
|
for (const testGroup of nodeTest) {
|
|
found = found ||
|
|
testGroup.tests.some(
|
|
test => minimatch(
|
|
test.modelUrl,
|
|
path.join('**', testCaseName, '*.+(onnx|ort)').replace(/\\/g, '/'),
|
|
));
|
|
}
|
|
if (!found) {
|
|
throw new Error(`node model test case '${testCaseName}' in test list does not exist.`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const onnxTest = onnxTests.get(backend);
|
|
if (onnxTest) {
|
|
const onnxModelTests = onnxTest.tests.map(i => i.name);
|
|
for (const testCase of testlist[backend].onnx) {
|
|
const testCaseName = typeof testCase === 'string' ? testCase : testCase.name;
|
|
if (onnxModelTests.indexOf(testCaseName) === -1) {
|
|
throw new Error(`onnx model test case '${testCaseName}' in test list does not exist.`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const opTest = opTests.get(backend);
|
|
if (opTest) {
|
|
const opTests = opTest.map(i => i.name);
|
|
for (const testCase of testlist[backend].ops) {
|
|
const testCaseName = typeof testCase === 'string' ? testCase : testCase.name;
|
|
if (opTests.indexOf(testCaseName) === -1) {
|
|
throw new Error(`operator test case '${testCaseName}' in test list does not exist.`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function loadNodeTests(backend: string, version: number): Test.ModelTestGroup {
|
|
return suiteFromFolder(
|
|
`node-opset_v${version}-${backend}`, path.join(TEST_DATA_MODEL_NODE_ROOT, `opset${version}`), backend,
|
|
testlist[backend].node);
|
|
}
|
|
|
|
function suiteFromFolder(
|
|
name: string, suiteRootFolder: string, backend: string,
|
|
testlist?: readonly Test.TestList.Test[]): Test.ModelTestGroup {
|
|
const sessions: Test.ModelTest[] = [];
|
|
const tests = fs.readdirSync(suiteRootFolder);
|
|
for (const test of tests) {
|
|
let condition: Test.Condition|undefined;
|
|
let times: number|undefined;
|
|
if (testlist) {
|
|
const matches = testlist.filter(
|
|
p => minimatch(
|
|
path.join(suiteRootFolder, test),
|
|
path.join('**', typeof p === 'string' ? p : p.name).replace(/\\/g, '/')));
|
|
if (matches.length === 0) {
|
|
times = 0;
|
|
} else if (matches.length === 1) {
|
|
const match = matches[0];
|
|
if (typeof match !== 'string') {
|
|
condition = match.condition;
|
|
}
|
|
} else {
|
|
throw new Error(`multiple testlist rules matches test: ${path.join(suiteRootFolder, test)}`);
|
|
}
|
|
}
|
|
sessions.push(modelTestFromFolder(path.resolve(suiteRootFolder, test), backend, condition, times));
|
|
}
|
|
return {name, tests: sessions};
|
|
}
|
|
|
|
function modelTestFromFolder(
|
|
testDataRootFolder: string, backend: string, condition?: Test.Condition, times?: number): Test.ModelTest {
|
|
if (times === 0) {
|
|
npmlog.verbose('TestRunnerCli.Init.Model', `Skip test data from folder: ${testDataRootFolder}`);
|
|
return {name: path.basename(testDataRootFolder), backend, modelUrl: '', cases: []};
|
|
}
|
|
|
|
let modelUrl: string|null = null;
|
|
let cases: Test.ModelTestCase[] = [];
|
|
|
|
npmlog.verbose('TestRunnerCli.Init.Model', `Start to prepare test data from folder: ${testDataRootFolder}`);
|
|
|
|
try {
|
|
for (const thisPath of fs.readdirSync(testDataRootFolder)) {
|
|
const thisFullPath = path.join(testDataRootFolder, thisPath);
|
|
const stat = fs.lstatSync(thisFullPath);
|
|
if (stat.isFile()) {
|
|
const ext = path.extname(thisPath);
|
|
if (ext.toLowerCase() === '.onnx' || ext.toLowerCase() === '.ort') {
|
|
if (modelUrl === null) {
|
|
modelUrl = path.join(TEST_DATA_BASE, path.relative(TEST_ROOT, thisFullPath));
|
|
if (FILE_CACHE_ENABLED && !fileCache[modelUrl] && stat.size <= FILE_CACHE_MAX_FILE_SIZE) {
|
|
fileCache[modelUrl] = bufferToBase64(fs.readFileSync(thisFullPath));
|
|
}
|
|
} else {
|
|
throw new Error('there are multiple model files under the folder specified');
|
|
}
|
|
}
|
|
} else if (stat.isDirectory()) {
|
|
const dataFiles: string[] = [];
|
|
for (const dataFile of fs.readdirSync(thisFullPath)) {
|
|
const dataFileFullPath = path.join(thisFullPath, dataFile);
|
|
const ext = path.extname(dataFile);
|
|
|
|
if (ext.toLowerCase() === '.pb') {
|
|
const dataFileUrl = path.join(TEST_DATA_BASE, path.relative(TEST_ROOT, dataFileFullPath));
|
|
dataFiles.push(dataFileUrl);
|
|
if (FILE_CACHE_ENABLED && !fileCache[dataFileUrl] &&
|
|
fs.lstatSync(dataFileFullPath).size <= FILE_CACHE_MAX_FILE_SIZE) {
|
|
fileCache[dataFileUrl] = bufferToBase64(fs.readFileSync(dataFileFullPath));
|
|
}
|
|
}
|
|
}
|
|
if (dataFiles.length > 0) {
|
|
cases.push({dataFiles, name: thisPath});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (modelUrl === null) {
|
|
throw new Error('there are no model file under the folder specified');
|
|
}
|
|
} catch (e) {
|
|
npmlog.error('TestRunnerCli.Init.Model', `Failed to prepare test data. Error: ${inspect(e)}`);
|
|
throw e;
|
|
}
|
|
|
|
const caseCount = cases.length;
|
|
if (times !== undefined) {
|
|
if (times > caseCount) {
|
|
for (let i = 0; cases.length < times; i++) {
|
|
const origin = cases[i % caseCount];
|
|
const duplicated = {name: `${origin.name} - copy ${Math.floor(i / caseCount)}`, dataFiles: origin.dataFiles};
|
|
cases.push(duplicated);
|
|
}
|
|
} else {
|
|
cases = cases.slice(0, times);
|
|
}
|
|
}
|
|
|
|
npmlog.verbose('TestRunnerCli.Init.Model', 'Finished preparing test data.');
|
|
npmlog.verbose('TestRunnerCli.Init.Model', '===============================================================');
|
|
npmlog.verbose('TestRunnerCli.Init.Model', ` Model file: ${modelUrl}`);
|
|
npmlog.verbose('TestRunnerCli.Init.Model', ` Backend: ${backend}`);
|
|
npmlog.verbose('TestRunnerCli.Init.Model', ` Test set(s): ${cases.length} (${caseCount})`);
|
|
npmlog.verbose('TestRunnerCli.Init.Model', '===============================================================');
|
|
|
|
return {name: path.basename(testDataRootFolder), condition, modelUrl, backend, cases};
|
|
}
|
|
|
|
function tryLocateModelTestFolder(searchPattern: string): string {
|
|
const folderCandidates: string[] = [];
|
|
// 1 - check whether search pattern is a directory
|
|
if (fs.existsSync(searchPattern) && fs.lstatSync(searchPattern).isDirectory()) {
|
|
folderCandidates.push(searchPattern);
|
|
}
|
|
|
|
// 2 - check the globby result of searchPattern
|
|
// 3 - check the globby result of ONNX root combined with searchPattern
|
|
const globbyPattern =
|
|
[searchPattern, path.join(TEST_DATA_MODEL_NODE_ROOT, '**', searchPattern).replace(/\\/g, '/')];
|
|
// 4 - check the globby result of NODE root combined with opset versions and searchPattern
|
|
globbyPattern.push(...DEFAULT_OPSET_VERSIONS.map(
|
|
v => path.join(TEST_DATA_MODEL_NODE_ROOT, `v${v}`, '**', searchPattern).replace(/\\/g, '/')));
|
|
|
|
folderCandidates.push(...globbySync(globbyPattern, {onlyDirectories: true, absolute: true}));
|
|
|
|
// pick the first folder that matches the pattern
|
|
for (const folderCandidate of folderCandidates) {
|
|
const modelCandidates = globbySync('*.{onnx,ort}', {onlyFiles: true, cwd: folderCandidate});
|
|
if (modelCandidates && modelCandidates.length === 1) {
|
|
return folderCandidate;
|
|
}
|
|
}
|
|
|
|
throw new Error(`no model folder found: ${searchPattern}`);
|
|
}
|
|
|
|
function loadOpTests(backend: string): Test.OperatorTestGroup[] {
|
|
const groups: Test.OperatorTestGroup[] = [];
|
|
for (const thisPath of fs.readdirSync(TEST_DATA_OP_ROOT)) {
|
|
const thisFullPath = path.join(TEST_DATA_OP_ROOT, thisPath);
|
|
const stat = fs.lstatSync(thisFullPath);
|
|
const ext = path.extname(thisFullPath);
|
|
if (stat.isFile() && (ext === '.json' || ext === '.jsonc')) {
|
|
const skip = testlist[backend].ops.indexOf(thisPath) === -1;
|
|
groups.push(opTestFromManifest(thisFullPath, backend, skip));
|
|
}
|
|
}
|
|
|
|
return groups;
|
|
}
|
|
|
|
function opTestFromManifest(manifestFile: string, backend: string, skip = false): Test.OperatorTestGroup {
|
|
let tests: Test.OperatorTest[] = [];
|
|
const filePath = path.resolve(process.cwd(), manifestFile);
|
|
if (skip) {
|
|
npmlog.verbose('TestRunnerCli.Init.Op', `Skip test data from manifest file: ${filePath}`);
|
|
} else {
|
|
npmlog.verbose('TestRunnerCli.Init.Op', `Start to prepare test data from manifest file: ${filePath}`);
|
|
const jsonWithComments = fs.readFileSync(filePath).toString();
|
|
const json = stripJsonComments(jsonWithComments, {whitespace: true});
|
|
tests = JSON.parse(json) as Test.OperatorTest[];
|
|
// field 'verbose' and 'backend' is not set
|
|
for (const test of tests) {
|
|
test.backend = backend;
|
|
}
|
|
npmlog.verbose('TestRunnerCli.Init.Op', 'Finished preparing test data.');
|
|
npmlog.verbose('TestRunnerCli.Init.Op', '===============================================================');
|
|
npmlog.verbose('TestRunnerCli.Init.Op', ` Test Group: ${path.relative(TEST_DATA_OP_ROOT, filePath)}`);
|
|
npmlog.verbose('TestRunnerCli.Init.Op', ` Backend: ${backend}`);
|
|
npmlog.verbose('TestRunnerCli.Init.Op', ` Test case(s): ${tests.length}`);
|
|
npmlog.verbose('TestRunnerCli.Init.Op', '===============================================================');
|
|
}
|
|
return {name: path.relative(TEST_DATA_OP_ROOT, filePath), tests};
|
|
}
|
|
|
|
function tryLocateOpTestManifest(searchPattern: string): string {
|
|
for (const manifestCandidate of globbySync(
|
|
[
|
|
searchPattern, path.join(TEST_DATA_OP_ROOT, '**', searchPattern).replace(/\\/g, '/'),
|
|
path.join(TEST_DATA_OP_ROOT, '**', searchPattern + '.json').replace(/\\/g, '/'),
|
|
path.join(TEST_DATA_OP_ROOT, '**', searchPattern + '.jsonc').replace(/\\/g, '/')
|
|
],
|
|
{onlyFiles: true, absolute: true, cwd: TEST_ROOT})) {
|
|
return manifestCandidate;
|
|
}
|
|
|
|
throw new Error(`no OP test manifest found: ${searchPattern}`);
|
|
}
|
|
|
|
function run(config: Test.Config) {
|
|
// STEP 1. write file cache to testdata-file-cache-*.json
|
|
npmlog.info('TestRunnerCli.Run', '(1/4) Writing file cache to file: testdata-file-cache-*.json ...');
|
|
const fileCacheUrls = saveFileCache(fileCache);
|
|
if (fileCacheUrls.length > 0) {
|
|
config.fileCacheUrls = fileCacheUrls;
|
|
}
|
|
npmlog.info(
|
|
'TestRunnerCli.Run',
|
|
`(1/4) Writing file cache to file: testdata-file-cache-*.json ... ${
|
|
fileCacheUrls.length > 0 ? `DONE, ${fileCacheUrls.length} file(s) generated` : 'SKIPPED'}`);
|
|
|
|
// STEP 2. write the config to testdata-config.json
|
|
npmlog.info('TestRunnerCli.Run', '(2/4) Writing config to file: testdata-config.json ...');
|
|
saveConfig(config);
|
|
npmlog.info('TestRunnerCli.Run', '(2/4) Writing config to file: testdata-config.json ... DONE');
|
|
|
|
// STEP 3. generate bundle
|
|
npmlog.info('TestRunnerCli.Run', '(3/4) Running build to generate bundle...');
|
|
const buildCommand = `node ${path.join(__dirname, 'build')}`;
|
|
const buildArgs = [`--bundle-mode=${args.env === 'node' ? 'node' : args.bundleMode}`];
|
|
if (args.backends.indexOf('wasm') === -1) {
|
|
buildArgs.push('--no-wasm');
|
|
}
|
|
npmlog.info('TestRunnerCli.Run', `CMD: ${buildCommand} ${buildArgs.join(' ')}`);
|
|
const build = spawnSync(buildCommand, buildArgs, {shell: true, stdio: 'inherit'});
|
|
if (build.status !== 0) {
|
|
console.error(build.error);
|
|
process.exit(build.status === null ? undefined : build.status);
|
|
}
|
|
npmlog.info('TestRunnerCli.Run', '(3/4) Running build to generate bundle... DONE');
|
|
|
|
if (args.env === 'node') {
|
|
// STEP 5. run tsc and run mocha
|
|
npmlog.info('TestRunnerCli.Run', '(4/4) Running tsc...');
|
|
const tsc = spawnSync('npx', ['tsc'], {shell: true, stdio: 'inherit'});
|
|
if (tsc.status !== 0) {
|
|
console.error(tsc.error);
|
|
process.exit(tsc.status === null ? undefined : tsc.status);
|
|
}
|
|
npmlog.info('TestRunnerCli.Run', '(4/4) Running tsc... DONE');
|
|
|
|
npmlog.info('TestRunnerCli.Run', '(4/4) Running mocha...');
|
|
const mochaArgs = ['mocha', path.join(TEST_ROOT, 'test-main'), `--timeout ${args.debug ? 9999999 : 60000}`];
|
|
npmlog.info('TestRunnerCli.Run', `CMD: npx ${mochaArgs.join(' ')}`);
|
|
const mocha = spawnSync('npx', mochaArgs, {shell: true, stdio: 'inherit'});
|
|
if (mocha.status !== 0) {
|
|
console.error(mocha.error);
|
|
process.exit(mocha.status === null ? undefined : mocha.status);
|
|
}
|
|
npmlog.info('TestRunnerCli.Run', '(4/4) Running mocha... DONE');
|
|
|
|
} else {
|
|
// STEP 5. use Karma to run test
|
|
npmlog.info('TestRunnerCli.Run', '(4/4) Running karma to start test runner...');
|
|
const browser = getBrowserNameFromEnv(
|
|
args.env,
|
|
args.bundleMode === 'perf' ? 'perf' :
|
|
args.debug ? 'debug' :
|
|
'test');
|
|
const karmaArgs = ['karma', 'start', `--browsers ${browser}`];
|
|
if (args.debug) {
|
|
karmaArgs.push('--log-level info --timeout-mocha 9999999');
|
|
} else {
|
|
karmaArgs.push('--single-run');
|
|
}
|
|
if (args.noSandbox) {
|
|
karmaArgs.push('--no-sandbox');
|
|
}
|
|
karmaArgs.push(`--bundle-mode=${args.bundleMode}`);
|
|
if (browser === 'Edge') {
|
|
// There are currently 2 Edge browser launchers:
|
|
// - karma-edge-launcher: used to launch the old Edge browser
|
|
// - karma-chromium-edge-launcher: used to launch the new chromium-kernel Edge browser
|
|
//
|
|
|
|
// Those 2 plugins cannot be loaded at the same time, so we need to determine which launchers to use.
|
|
// - If we use 'karma-edge-launcher', no plugins config need to be set.
|
|
// - If we use 'karma-chromium-edge-launcher', we need to:
|
|
// - add plugin "@chiragrupani/karma-chromium-edge-launcher" explicitly, because it does not match the
|
|
// default plugins config "^karma-.*"
|
|
// - remove "karma-edge-launcher".
|
|
|
|
// check if we have the latest Edge installed:
|
|
if (os.platform() === 'darwin' ||
|
|
(os.platform() === 'win32' &&
|
|
require('@chiragrupani/karma-chromium-edge-launcher/dist/Utilities').default.GetEdgeExe('Edge') !== '')) {
|
|
// use "@chiragrupani/karma-chromium-edge-launcher"
|
|
karmaArgs.push(
|
|
'--karma-plugins=@chiragrupani/karma-chromium-edge-launcher',
|
|
'--karma-plugins=(?!karma-edge-launcher$)karma-*');
|
|
} else {
|
|
// use "karma-edge-launcher"
|
|
|
|
// == Special treatment to Microsoft Edge ==
|
|
//
|
|
// == Edge's Auto Recovery Feature ==
|
|
// when Edge starts, if it found itself was terminated forcely last time, it always recovers all previous
|
|
// pages. this always happen in Karma because `karma-edge-launcher` uses `taskkill` command to kill Edge every
|
|
// time.
|
|
//
|
|
// == The Problem ==
|
|
// every time when a test is completed, it will be added to the recovery page list.
|
|
// if we run the test 100 times, there will be 100 previous tabs when we launch Edge again.
|
|
// this run out of resources quickly and fails the futher test.
|
|
// and it cannot recover by itself because every time it is terminated forcely or crashes.
|
|
// and the auto recovery feature has no way to disable by configuration/commandline/registry
|
|
//
|
|
// == The Solution ==
|
|
// for Microsoft Edge, we should clean up the previous active page before each run
|
|
// delete the files stores in the specific folder to clean up the recovery page list.
|
|
// see also: https://www.laptopmag.com/articles/edge-browser-stop-tab-restore
|
|
const deleteEdgeActiveRecoveryCommand =
|
|
// eslint-disable-next-line max-len
|
|
'del /F /Q % LOCALAPPDATA %\\Packages\\Microsoft.MicrosoftEdge_8wekyb3d8bbwe\\AC\\MicrosoftEdge\\User\\Default\\Recovery\\Active\\*';
|
|
npmlog.info('TestRunnerCli.Run', `CMD: ${deleteEdgeActiveRecoveryCommand}`);
|
|
spawnSync(deleteEdgeActiveRecoveryCommand, {shell: true, stdio: 'inherit'});
|
|
}
|
|
}
|
|
npmlog.info('TestRunnerCli.Run', `CMD: npx ${karmaArgs.join(' ')}`);
|
|
const karma = spawnSync('npx', karmaArgs, {shell: true, stdio: 'inherit'});
|
|
if (karma.status !== 0) {
|
|
console.error(karma.error);
|
|
process.exit(karma.status === null ? undefined : karma.status);
|
|
}
|
|
npmlog.info('TestRunnerCli.Run', '(4/4) Running karma to start test runner... DONE');
|
|
}
|
|
}
|
|
|
|
function saveFileCache(fileCache: Test.FileCache) {
|
|
const fileCacheUrls: string[] = [];
|
|
let currentIndex = 0;
|
|
let currentCache: Test.FileCache = {};
|
|
let currentContentTotalSize = 0;
|
|
for (const key in fileCache) {
|
|
const content = fileCache[key];
|
|
if (currentContentTotalSize > FILE_CACHE_SPLIT_SIZE) {
|
|
fileCacheUrls.push(saveOneFileCache(currentIndex, currentCache));
|
|
currentContentTotalSize = 0;
|
|
currentIndex++;
|
|
currentCache = {};
|
|
}
|
|
currentCache[key] = content;
|
|
currentContentTotalSize += key.length + content.length;
|
|
}
|
|
if (currentContentTotalSize > 0) {
|
|
fileCacheUrls.push(saveOneFileCache(currentIndex, currentCache));
|
|
}
|
|
return fileCacheUrls;
|
|
}
|
|
|
|
function saveOneFileCache(index: number, fileCache: Test.FileCache) {
|
|
fs.writeFileSync(path.join(TEST_ROOT, `./testdata-file-cache-${index}.json`), JSON.stringify(fileCache));
|
|
return path.join(TEST_DATA_BASE, `./testdata-file-cache-${index}.json`);
|
|
}
|
|
|
|
function saveConfig(config: Test.Config) {
|
|
fs.writeJSONSync(path.join(TEST_ROOT, './testdata-config.json'), config);
|
|
}
|
|
|
|
function getBrowserNameFromEnv(env: TestRunnerCliArgs['env'], mode: 'debug'|'perf'|'test') {
|
|
switch (env) {
|
|
case 'chrome':
|
|
return selectChromeBrowser(mode);
|
|
case 'edge':
|
|
return 'Edge';
|
|
case 'firefox':
|
|
return 'Firefox';
|
|
case 'electron':
|
|
return 'Electron';
|
|
case 'safari':
|
|
return 'Safari';
|
|
case 'bs':
|
|
return process.env.ORT_WEB_TEST_BS_BROWSERS!;
|
|
default:
|
|
throw new Error(`env "${env}" not supported.`);
|
|
}
|
|
}
|
|
|
|
function selectChromeBrowser(mode: 'debug'|'perf'|'test') {
|
|
switch (mode) {
|
|
case 'debug':
|
|
return 'ChromeDebug';
|
|
case 'perf':
|
|
return 'ChromePerf';
|
|
default:
|
|
return 'ChromeTest';
|
|
}
|
|
}
|
|
}
|
|
|
|
void main();
|