onnxruntime/js/web/script/test-runner-cli.ts

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

/* eslint-disable guard-for-in */
/* eslint-disable @typescript-eslint/no-use-before-define */

import { spawnSync } from 'child_process';
import * as fs from 'fs-extra';
import { default as minimatch } from 'minimatch';
import npmlog from 'npmlog';
import * as os from 'os';
import * as path from 'path';
import { inspect } from 'util';

import { onnx } from '../lib/onnxjs/ort-schema/protobuf/onnx';
import { bufferToBase64 } from '../test/test-shared';
import { Test } from '../test/test-types';

import { parseTestRunnerCliArgs, TestRunnerCliArgs } from './test-runner-cli-args';

async function main() {
  // use dynamic import so that we can use ESM only libraries in commonJS.
  const { globbySync } = await import('globby');
  const stripJsonComments = (await import('strip-json-comments')).default;

  npmlog.info('TestRunnerCli', 'Initializing...');

  const args = parseTestRunnerCliArgs(process.argv.slice(2));

  npmlog.verbose('TestRunnerCli.Init.Config', inspect(args));

  const DIST_ROOT = path.join(__dirname, '..', 'dist');
  const TEST_ROOT = path.join(__dirname, '..', 'test');
  const TEST_DATA_MODEL_NODE_ROOT = path.join(TEST_ROOT, 'data', 'node');
  const TEST_DATA_OP_ROOT = path.join(TEST_ROOT, 'data', 'ops');

  const TEST_DATA_BASE = args.env === 'node' ? TEST_ROOT : '/base/test/';

  npmlog.verbose('TestRunnerCli.Init', 'Ensure test data folder...');
  fs.ensureSymlinkSync(path.join(__dirname, '../../test/data/node'), TEST_DATA_MODEL_NODE_ROOT, 'junction');
  npmlog.verbose('TestRunnerCli.Init', 'Ensure test data folder... DONE');

  let testlist: Test.TestList;
  const shouldLoadSuiteTestData = args.mode === 'suite0' || args.mode === 'suite1';
  if (shouldLoadSuiteTestData) {
    npmlog.verbose('TestRunnerCli.Init', 'Loading testlist...');

    // The following is a list of unittests for already implemented operators.
    // Modify this list to control what node tests to run.
    const jsonWithComments = fs.readFileSync(path.resolve(TEST_ROOT, './suite-test-list.jsonc')).toString();
    const json = stripJsonComments(jsonWithComments, { whitespace: true });
    testlist = JSON.parse(json) as Test.TestList;
    npmlog.verbose('TestRunnerCli.Init', 'Loading testlist... DONE');
  }

  // The default backends and opset version lists. Those will be used in suite tests.
  const DEFAULT_BACKENDS: readonly TestRunnerCliArgs.Backend[] =
    args.env === 'node' ? ['cpu', 'wasm'] : ['wasm', 'webgl', 'webgpu', 'webnn'];
  const DEFAULT_OPSET_VERSIONS = fs
    .readdirSync(TEST_DATA_MODEL_NODE_ROOT, { withFileTypes: true })
    .filter((dir) => dir.isDirectory() && dir.name.startsWith('opset'))
    .map((dir) => dir.name.slice(5));
  const MAX_OPSET_VERSION = Math.max(...DEFAULT_OPSET_VERSIONS.map((v) => Number.parseInt(v, 10)));

  const FILE_CACHE_ENABLED = args.fileCache; // whether to enable file cache
  const FILE_CACHE_MAX_FILE_SIZE = 1 * 1024 * 1024; // The max size of the file that will be put into file cache
  const FILE_CACHE_SPLIT_SIZE = 4 * 1024 * 1024; // The min size of the cache file
  const fileCache: Test.FileCache = {};

  const nodeTests = new Map<string, Test.ModelTestGroup[]>();
  const onnxTests = new Map<string, Test.ModelTestGroup>();
  const opTests = new Map<string, Test.OperatorTestGroup[]>();

  if (shouldLoadSuiteTestData) {
    npmlog.verbose('TestRunnerCli.Init', 'Loading test groups for suite test...');

    // collect all model test folders
    const allNodeTestsFolders = DEFAULT_OPSET_VERSIONS.map((version) => {
      const suiteRootFolder = path.join(TEST_DATA_MODEL_NODE_ROOT, `opset${version}`);
      if (!fs.existsSync(suiteRootFolder) || !fs.statSync(suiteRootFolder).isDirectory()) {
        throw new Error(`model test root folder '${suiteRootFolder}' does not exist.`);
      }
      return fs.readdirSync(suiteRootFolder).map((f) => `opset${version}/${f}`);
    }).flat();

    for (const backend of DEFAULT_BACKENDS) {
      if (args.backends.indexOf(backend) !== -1) {
        nodeTests.set(backend, loadNodeTests(backend, allNodeTestsFolders));
        opTests.set(backend, loadOpTests(backend));
      }
    }
  }

  if (shouldLoadSuiteTestData) {
    npmlog.verbose('TestRunnerCli.Init', 'Loading test groups for suite test... DONE');

    npmlog.verbose('TestRunnerCli.Init', 'Validate testlist...');
    validateTestList();
    npmlog.verbose('TestRunnerCli.Init', 'Validate testlist... DONE');
  }

  const modelTestGroups: Test.ModelTestGroup[] = [];
  const opTestGroups: Test.OperatorTestGroup[] = [];
  let unittest = false;

  npmlog.verbose('TestRunnerCli.Init', 'Preparing test config...');
  switch (args.mode) {
    case 'suite0':
    case 'suite1':
      for (const backend of DEFAULT_BACKENDS) {
        if (args.backends.indexOf(backend) !== -1) {
          modelTestGroups.push(...nodeTests.get(backend)!); // model test : node
          opTestGroups.push(...opTests.get(backend)!); // operator test
        }
      }
      if (args.mode === 'suite0') {
        unittest = true;
      }
      break;

    case 'model':
      if (!args.param) {
        throw new Error("the test folder should be specified in mode 'node'");
      } else {
        const testFolderSearchPattern = args.param;
        const testFolder = tryLocateModelTestFolder(testFolderSearchPattern);
        for (const b of args.backends) {
          modelTestGroups.push({
            name: testFolder,
            tests: [modelTestFromFolder(testFolder, b, undefined, args.times)],
          });
        }
      }
      break;

    case 'unittest':
      unittest = true;
      break;

    case 'op':
      if (!args.param) {
        throw new Error("the test manifest should be specified in mode 'op'");
      } else {
        const manifestFileSearchPattern = args.param;
        const manifestFile = tryLocateOpTestManifest(manifestFileSearchPattern);
        for (const b of args.backends) {
          opTestGroups.push(opTestFromManifest(manifestFile, b));
        }
      }
      break;
    default:
      throw new Error(`unsupported mode '${args.mode}'`);
  }

  npmlog.verbose('TestRunnerCli.Init', 'Preparing test config... DONE');

  npmlog.info('TestRunnerCli', 'Initialization completed. Start to run tests...');
  run({
    unittest,
    model: modelTestGroups,
    op: opTestGroups,
    log: args.logConfig,
    profile: args.profile,
    downloadModel: args.downloadModel,
    options: {
      sessionOptions: {
        graphOptimizationLevel: args.graphOptimizationLevel,
        optimizedModelFilePath: args.optimizedModelFilePath,
      },
      debug: args.debug,
      cpuOptions: args.cpuOptions,
      webglOptions: args.webglOptions,
      webnnOptions: args.webnnOptions,
      wasmOptions: args.wasmOptions,
      globalEnvFlags: args.globalEnvFlags,
    },
  });
  npmlog.info('TestRunnerCli', 'Tests completed successfully');

  function validateTestList() {
    for (const backend of DEFAULT_BACKENDS) {
      const nodeTest = nodeTests.get(backend);
      if (nodeTest) {
        for (const testCase of testlist[backend].node) {
          const testCaseName = typeof testCase === 'string' ? testCase : testCase.name;
          let found = false;
          for (const testGroup of nodeTest) {
            found ||=
              minimatch.match(
                testGroup.tests.map((test) => test.modelUrl).filter((url) => url !== ''),
                path.join('**', testCaseName, '*.+(onnx|ort)').replace(/\\/g, '/'),
                { matchBase: true },
              ).length > 0;
          }
          if (!found) {
            throw new Error(`node model test case '${testCaseName}' in test list does not exist.`);
          }
        }
      }

      const onnxTest = onnxTests.get(backend);
      if (onnxTest) {
        const onnxModelTests = onnxTest.tests.map((i) => i.name);
        for (const testCase of testlist[backend].onnx) {
          const testCaseName = typeof testCase === 'string' ? testCase : testCase.name;
          if (onnxModelTests.indexOf(testCaseName) === -1) {
            throw new Error(`onnx model test case '${testCaseName}' in test list does not exist.`);
          }
        }
      }

      const opTest = opTests.get(backend);
      if (opTest) {
        const opTests = opTest.map((i) => i.name);
        for (const testCase of testlist[backend].ops) {
          const testCaseName = typeof testCase === 'string' ? testCase : testCase.name;
          if (opTests.indexOf(testCaseName) === -1) {
            throw new Error(`operator test case '${testCaseName}' in test list does not exist.`);
          }
        }
      }
    }
  }

  function loadNodeTests(backend: string, allFolders: string[]): Test.ModelTestGroup[] {
    const allTests = testlist[backend]?.node;

    // key is folder name, value is test index array
    const folderTestMatchCount = new Map<string, number[]>(allFolders.map((f) => [f, []]));
    // key is test category, value is a list of model test
    const opsetTests = new Map<string, Test.ModelTest[]>();

    allTests.forEach((test, i) => {
      const testName = typeof test === 'string' ? test : test.name;
      const matches = minimatch.match(allFolders, path.join('**', testName).replace(/\\/g, '/'));
      matches.forEach((m) => folderTestMatchCount.get(m)!.push(i));
    });

    for (const folder of allFolders) {
      const testIds = folderTestMatchCount.get(folder);
      const times = testIds ? testIds.length : 0;
      if (times > 1) {
        throw new Error(`multiple testlist rules matches test: ${path.join(TEST_DATA_MODEL_NODE_ROOT, folder)}`);
      }

      const test = testIds && testIds.length > 0 ? allTests[testIds[0]] : undefined;
      const platformCondition = test && typeof test !== 'string' ? test.platformCondition : undefined;

      const opsetVersion = folder.split('/')[0];
      const category = `node-${opsetVersion}-${backend}`;
      let modelTests = opsetTests.get(category);
      if (!modelTests) {
        modelTests = [];
        opsetTests.set(category, modelTests);
      }
      modelTests.push(
        modelTestFromFolder(path.resolve(TEST_DATA_MODEL_NODE_ROOT, folder), backend, platformCondition, times),
      );
    }

    return Array.from(opsetTests.keys()).map((category) => ({ name: category, tests: opsetTests.get(category)! }));
  }

  function modelTestFromFolder(
    testDataRootFolder: string,
    backend: string,
    platformCondition?: Test.PlatformCondition,
    times?: number,
  ): Test.ModelTest {
    if (times === 0) {
      npmlog.verbose('TestRunnerCli.Init.Model', `Skip test data from folder: ${testDataRootFolder}`);
      return {
        name: path.basename(testDataRootFolder),
        backend,
        modelUrl: '',
        cases: [],
        ioBinding: args.ioBindingMode,
      };
    }

    let modelUrl: string | null = null;
    let cases: Test.ModelTestCase[] = [];
    let externalData: Array<{ data: string; path: string }> | undefined;

    npmlog.verbose('TestRunnerCli.Init.Model', `Start to prepare test data from folder: ${testDataRootFolder}`);

    try {
      const maybeExternalDataFiles: Array<[fileNameWithoutExtension: string, size: number]> = [];
      for (const thisPath of fs.readdirSync(testDataRootFolder)) {
        const thisFullPath = path.join(testDataRootFolder, thisPath);
        const stat = fs.lstatSync(thisFullPath);
        if (stat.isFile()) {
          const ext = path.extname(thisPath);
          if (ext.toLowerCase() === '.onnx' || ext.toLowerCase() === '.ort') {
            if (modelUrl === null) {
              modelUrl = path.join(TEST_DATA_BASE, path.relative(TEST_ROOT, thisFullPath));
              if (FILE_CACHE_ENABLED && !fileCache[modelUrl] && stat.size <= FILE_CACHE_MAX_FILE_SIZE) {
                fileCache[modelUrl] = bufferToBase64(fs.readFileSync(thisFullPath));
              }
            } else {
              throw new Error('there are multiple model files under the folder specified');
            }
          } else {
            maybeExternalDataFiles.push([path.parse(thisPath).name, stat.size]);
          }
        } else if (stat.isDirectory()) {
          const dataFiles: string[] = [];
          for (const dataFile of fs.readdirSync(thisFullPath)) {
            const dataFileFullPath = path.join(thisFullPath, dataFile);
            const ext = path.extname(dataFile);

            if (ext.toLowerCase() === '.pb') {
              const dataFileUrl = path.join(TEST_DATA_BASE, path.relative(TEST_ROOT, dataFileFullPath));
              dataFiles.push(dataFileUrl);
              if (
                FILE_CACHE_ENABLED &&
                !fileCache[dataFileUrl] &&
                fs.lstatSync(dataFileFullPath).size <= FILE_CACHE_MAX_FILE_SIZE
              ) {
                fileCache[dataFileUrl] = bufferToBase64(fs.readFileSync(dataFileFullPath));
              }
            }
          }
          if (dataFiles.length > 0) {
            cases.push({ dataFiles, name: thisPath });
          }
        }
      }

      if (modelUrl === null) {
        throw new Error('there are no model file under the folder specified');
      }
      // for performance consideration, we do not parse every model. when we think it's likely to have external
      // data, we will parse it. We think it's "likely" when one of the following conditions is met:
      // 1. any file in the same folder has the similar file name as the model file
      //    (e.g., model file is "model_abc.onnx", and there is a file "model_abc.pb" or "model_abc.onnx.data")
      // 2. the file size is larger than 1GB
      const likelyToHaveExternalData = maybeExternalDataFiles.some(
        ([fileNameWithoutExtension, size]) =>
          path.basename(modelUrl!).startsWith(fileNameWithoutExtension) || size >= 1 * 1024 * 1024 * 1024,
      );
      if (likelyToHaveExternalData) {
        const model = onnx.ModelProto.decode(fs.readFileSync(path.join(testDataRootFolder, path.basename(modelUrl!))));
        const externalDataPathSet = new Set<string>();
        for (const initializer of model.graph!.initializer!) {
          if (initializer.externalData) {
            for (const data of initializer.externalData) {
              if (data.key === 'location') {
                externalDataPathSet.add(data.value!);
              }
            }
          }
        }
        externalData = [];
        const externalDataPaths = [...externalDataPathSet];
        for (const dataPath of externalDataPaths) {
          const fullPath = path.resolve(testDataRootFolder, dataPath);
          const url = path.join(TEST_DATA_BASE, path.relative(TEST_ROOT, fullPath));
          externalData.push({ data: url, path: dataPath });
        }
      }
    } catch (e) {
      npmlog.error('TestRunnerCli.Init.Model', `Failed to prepare test data. Error: ${inspect(e)}`);
      throw e;
    }

    const caseCount = cases.length;
    if (times !== undefined) {
      if (times > caseCount) {
        for (let i = 0; cases.length < times; i++) {
          const origin = cases[i % caseCount];
          const duplicated = {
            name: `${origin.name} - copy ${Math.floor(i / caseCount)}`,
            dataFiles: origin.dataFiles,
          };
          cases.push(duplicated);
        }
      } else {
        cases = cases.slice(0, times);
      }
    }

    let ioBinding: Test.IOBindingMode;
    if (!['webgpu', 'webnn'].includes(backend) && args.ioBindingMode !== 'none') {
      npmlog.warn(
        'TestRunnerCli.Init.Model',
        `Ignoring IO Binding Mode "${args.ioBindingMode}" for backend "${backend}".`,
      );
      ioBinding = 'none';
    } else {
      ioBinding = args.ioBindingMode;
    }

    npmlog.verbose('TestRunnerCli.Init.Model', 'Finished preparing test data.');
    npmlog.verbose('TestRunnerCli.Init.Model', '===============================================================');
    npmlog.verbose('TestRunnerCli.Init.Model', ` Model file: ${modelUrl}`);
    npmlog.verbose('TestRunnerCli.Init.Model', ` Backend: ${backend}`);
    npmlog.verbose('TestRunnerCli.Init.Model', ` Test set(s): ${cases.length} (${caseCount})`);
    if (externalData) {
      npmlog.verbose('TestRunnerCli.Init.Model', ` External data: ${externalData.length}`);
      for (const data of externalData) {
        npmlog.verbose('TestRunnerCli.Init.Model', `  - ${data.path}`);
      }
    }
    npmlog.verbose('TestRunnerCli.Init.Model', '===============================================================');

    return {
      name: path.basename(testDataRootFolder),
      platformCondition,
      modelUrl,
      backend,
      cases,
      ioBinding,
      externalData,
    };
  }

  function tryLocateModelTestFolder(searchPattern: string): string {
    const folderCandidates: string[] = [];
    // 1 - check whether search pattern is a directory
    if (fs.existsSync(searchPattern) && fs.lstatSync(searchPattern).isDirectory()) {
      folderCandidates.push(searchPattern);
    }

    // 2 - check the globby result of searchPattern
    // 3 - check the globby result of ONNX root combined with searchPattern
    const globbyPattern = [
      searchPattern,
      path.join(TEST_DATA_MODEL_NODE_ROOT, '**', searchPattern).replace(/\\/g, '/'),
    ];
    // 4 - check the globby result of NODE root combined with opset versions and searchPattern
    globbyPattern.push(
      ...DEFAULT_OPSET_VERSIONS.map((v) =>
        path.join(TEST_DATA_MODEL_NODE_ROOT, `opset${v}`, '**', searchPattern).replace(/\\/g, '/'),
      ),
    );

    folderCandidates.push(...globbySync(globbyPattern, { onlyDirectories: true, absolute: true }));

    // pick the first folder that matches the pattern
    for (const folderCandidate of folderCandidates) {
      const modelCandidates = globbySync('*.{onnx,ort}', { onlyFiles: true, cwd: folderCandidate });
      if (modelCandidates && modelCandidates.length === 1) {
        return folderCandidate;
      }
    }

    throw new Error(`no model folder found: ${searchPattern}`);
  }

  function loadOpTests(backend: string): Test.OperatorTestGroup[] {
    const groups: Test.OperatorTestGroup[] = [];
    for (const thisPath of fs.readdirSync(TEST_DATA_OP_ROOT)) {
      const thisFullPath = path.join(TEST_DATA_OP_ROOT, thisPath);
      const stat = fs.lstatSync(thisFullPath);
      const ext = path.extname(thisFullPath);
      if (stat.isFile() && (ext === '.json' || ext === '.jsonc')) {
        const skip = testlist[backend].ops.indexOf(thisPath) === -1;
        groups.push(opTestFromManifest(thisFullPath, backend, skip));
      }
    }

    return groups;
  }

  function opTestFromManifest(manifestFile: string, backend: string, skip = false): Test.OperatorTestGroup {
    let tests: Test.OperatorTest[] = [];
    const filePath = path.resolve(process.cwd(), manifestFile);
    if (skip) {
      npmlog.verbose('TestRunnerCli.Init.Op', `Skip test data from manifest file: ${filePath}`);
    } else {
      npmlog.verbose('TestRunnerCli.Init.Op', `Start to prepare test data from manifest file: ${filePath}`);
      const jsonWithComments = fs.readFileSync(filePath).toString();
      const json = stripJsonComments(jsonWithComments, { whitespace: true });
      tests = JSON.parse(json) as Test.OperatorTest[];
      // field 'verbose' and 'backend' is not set
      for (const test of tests) {
        test.backend = backend;
        test.opset = test.opset || { domain: '', version: MAX_OPSET_VERSION };
        if (backend !== 'webgpu' && args.ioBindingMode !== 'none') {
          npmlog.warn(
            'TestRunnerCli.Init.Op',
            `Ignoring IO Binding Mode "${args.ioBindingMode}" for backend "${backend}".`,
          );
          test.ioBinding = 'none';
        } else {
          test.ioBinding = args.ioBindingMode;
        }
      }
      npmlog.verbose('TestRunnerCli.Init.Op', 'Finished preparing test data.');
      npmlog.verbose('TestRunnerCli.Init.Op', '===============================================================');
      npmlog.verbose('TestRunnerCli.Init.Op', ` Test Group: ${path.relative(TEST_DATA_OP_ROOT, filePath)}`);
      npmlog.verbose('TestRunnerCli.Init.Op', ` Backend: ${backend}`);
      npmlog.verbose('TestRunnerCli.Init.Op', ` Test case(s): ${tests.length}`);
      npmlog.verbose('TestRunnerCli.Init.Op', '===============================================================');
    }
    return { name: path.relative(TEST_DATA_OP_ROOT, filePath), tests };
  }

  function tryLocateOpTestManifest(searchPattern: string): string {
    for (const manifestCandidate of globbySync(
      [
        searchPattern,
        path.join(TEST_DATA_OP_ROOT, '**', searchPattern).replace(/\\/g, '/'),
        path.join(TEST_DATA_OP_ROOT, '**', searchPattern + '.json').replace(/\\/g, '/'),
        path.join(TEST_DATA_OP_ROOT, '**', searchPattern + '.jsonc').replace(/\\/g, '/'),
      ],
      { onlyFiles: true, absolute: true, cwd: TEST_ROOT },
    )) {
      return manifestCandidate;
    }

    throw new Error(`no OP test manifest found: ${searchPattern}`);
  }

  function run(config: Test.Config) {
    // STEP 1. write file cache to testdata-file-cache-*.json
    npmlog.info('TestRunnerCli.Run', '(1/4) Writing file cache to file: testdata-file-cache-*.json ...');
    const fileCacheUrls = saveFileCache(fileCache);
    if (fileCacheUrls.length > 0) {
      config.fileCacheUrls = fileCacheUrls;
    }
    npmlog.info(
      'TestRunnerCli.Run',
      `(1/4) Writing file cache to file: testdata-file-cache-*.json ... ${
        fileCacheUrls.length > 0 ? `DONE, ${fileCacheUrls.length} file(s) generated` : 'SKIPPED'
      }`,
    );

    // STEP 2. write the config to testdata-config.json
    npmlog.info('TestRunnerCli.Run', '(2/4) Writing config to file: testdata-config.json ...');
    saveConfig(config);
    npmlog.info('TestRunnerCli.Run', '(2/4) Writing config to file: testdata-config.json ... DONE');

    // STEP 3. generate bundle
    npmlog.info('TestRunnerCli.Run', '(3/4) Running build to generate bundle...');
    const buildCommand = `node ${path.join(__dirname, 'build')}`;
    const buildArgs = [`--bundle-mode=${args.env === 'node' ? 'node' : args.bundleMode}`];
    npmlog.info('TestRunnerCli.Run', `CMD: ${buildCommand} ${buildArgs.join(' ')}`);
    const build = spawnSync(buildCommand, buildArgs, { shell: true, stdio: 'inherit' });
    if (build.status !== 0) {
      console.error(build.error);
      process.exit(build.status === null ? undefined : build.status);
    }
    npmlog.info('TestRunnerCli.Run', '(3/4) Running build to generate bundle... DONE');

    if (args.env === 'node') {
      // STEP 5. run tsc and run mocha
      npmlog.info('TestRunnerCli.Run', '(4/4) Running tsc...');
      const tsc = spawnSync('npx', ['tsc'], { shell: true, stdio: 'inherit' });
      if (tsc.status !== 0) {
        console.error(tsc.error);
        process.exit(tsc.status === null ? undefined : tsc.status);
      }
      npmlog.info('TestRunnerCli.Run', '(4/4) Running tsc... DONE');

      npmlog.info('TestRunnerCli.Run', '(4/4) Running mocha...');
      const mochaArgs = [
        'mocha',
        '--timeout',
        `${args.debug ? 9999999 : 60000}`,
        '-r',
        path.join(DIST_ROOT, 'ort.node.min.js'),
        path.join(TEST_ROOT, 'test-main'),
      ];
      npmlog.info('TestRunnerCli.Run', `CMD: npx ${mochaArgs.join(' ')}`);
      const mocha = spawnSync('npx', mochaArgs, { shell: true, stdio: 'inherit' });
      if (mocha.status !== 0) {
        console.error(mocha.error);
        process.exit(mocha.status === null ? undefined : mocha.status);
      }
      npmlog.info('TestRunnerCli.Run', '(4/4) Running mocha... DONE');
    } else {
      // STEP 5. use Karma to run test
      npmlog.info('TestRunnerCli.Run', '(4/4) Running karma to start test runner...');
      const webgpu = args.backends.indexOf('webgpu') > -1;
      const webnn = args.backends.indexOf('webnn') > -1;
      const browser = getBrowserNameFromEnv(args.env);
      const karmaArgs = ['karma', 'start', `--browsers ${browser}`];
      const chromiumFlags = ['--enable-features=SharedArrayBuffer', ...args.chromiumFlags];
      if (args.bundleMode === 'dev' && !args.debug) {
        // use headless for 'test' mode (when 'perf' and 'debug' are OFF)
        chromiumFlags.push('--headless=new');
      }
      if (args.debug) {
        karmaArgs.push('--log-level info --timeout-mocha 9999999');
        chromiumFlags.push('--remote-debugging-port=9333');
      } else {
        karmaArgs.push('--single-run');
      }
      if (args.noSandbox) {
        karmaArgs.push('--no-sandbox');
      }

      // When using BrowserStack with Safari, we need NOT to use 'localhost' as the hostname.
      if (!(browser.startsWith('BS_') && browser.includes('Safari'))) {
        karmaArgs.push('--force-localhost');
      }
      if (webgpu) {
        // flag 'allow_unsafe_apis' is required to enable experimental features like fp16 and profiling inside pass.
        // flag 'use_dxc' is required to enable DXC compiler.
        chromiumFlags.push('--enable-dawn-features=allow_unsafe_apis,use_dxc');
      }
      if (webnn) {
        chromiumFlags.push('--enable-features=WebMachineLearningNeuralNetwork');
      }
      if (process.argv.includes('--karma-debug')) {
        karmaArgs.push('--log-level debug');
      }
      karmaArgs.push(`--bundle-mode=${args.bundleMode}`);
      if (args.userDataDir) {
        karmaArgs.push(`--user-data-dir="${args.userDataDir}"`);
      }
      karmaArgs.push(...chromiumFlags.map((flag) => `--chromium-flags=${flag}`));
      if (browser.startsWith('Edge')) {
        // There are currently 2 Edge browser launchers:
        //  - karma-edge-launcher: used to launch the old Edge browser
        //  - karma-chromium-edge-launcher: used to launch the new chromium-kernel Edge browser
        //

        // Those 2 plugins cannot be loaded at the same time, so we need to determine which launchers to use.
        //  - If we use 'karma-edge-launcher', no plugins config need to be set.
        //  - If we use 'karma-chromium-edge-launcher', we need to:
        //       - add plugin "@chiragrupani/karma-chromium-edge-launcher" explicitly, because it does not match the
        //         default plugins config "^karma-.*"
        //       - remove "karma-edge-launcher".

        // check if we have the latest Edge installed:
        if (
          os.platform() === 'darwin' ||
          (os.platform() === 'win32' &&
            require('@chiragrupani/karma-chromium-edge-launcher/dist/Utilities').default.GetEdgeExe('Edge') !== '')
        ) {
          // use "@chiragrupani/karma-chromium-edge-launcher"
          karmaArgs.push(
            '--karma-plugins=@chiragrupani/karma-chromium-edge-launcher',
            '--karma-plugins=(?!karma-edge-launcher$)karma-*',
          );
        } else {
          // use "karma-edge-launcher"

          // == Special treatment to Microsoft Edge ==
          //
          // == Edge's Auto Recovery Feature ==
          // when Edge starts, if it found itself was terminated forcely last time, it always recovers all previous
          // pages. this always happen in Karma because `karma-edge-launcher` uses `taskkill` command to kill Edge every
          // time.
          //
          // == The Problem ==
          // every time when a test is completed, it will be added to the recovery page list.
          // if we run the test 100 times, there will be 100 previous tabs when we launch Edge again.
          // this run out of resources quickly and fails the further test.
          // and it cannot recover by itself because every time it is terminated forcely or crashes.
          // and the auto recovery feature has no way to disable by configuration/commandline/registry
          //
          // == The Solution ==
          // for Microsoft Edge, we should clean up the previous active page before each run
          // delete the files stores in the specific folder to clean up the recovery page list.
          // see also: https://www.laptopmag.com/articles/edge-browser-stop-tab-restore
          const deleteEdgeActiveRecoveryCommand =
            // eslint-disable-next-line max-len
            'del /F /Q % LOCALAPPDATA %\\Packages\\Microsoft.MicrosoftEdge_8wekyb3d8bbwe\\AC\\MicrosoftEdge\\User\\Default\\Recovery\\Active\\*';
          npmlog.info('TestRunnerCli.Run', `CMD: ${deleteEdgeActiveRecoveryCommand}`);
          spawnSync(deleteEdgeActiveRecoveryCommand, { shell: true, stdio: 'inherit' });
        }
      }
      npmlog.info('TestRunnerCli.Run', `CMD: npx ${karmaArgs.join(' ')}`);
      const karma = spawnSync('npx', karmaArgs, { shell: true, stdio: 'inherit' });
      if (karma.status !== 0) {
        console.error(karma.error);
        process.exit(karma.status === null ? undefined : karma.status);
      }
      npmlog.info('TestRunnerCli.Run', '(4/4) Running karma to start test runner... DONE');
    }
  }

  function saveFileCache(fileCache: Test.FileCache) {
    const fileCacheUrls: string[] = [];
    let currentIndex = 0;
    let currentCache: Test.FileCache = {};
    let currentContentTotalSize = 0;
    for (const key in fileCache) {
      const content = fileCache[key];
      if (currentContentTotalSize > FILE_CACHE_SPLIT_SIZE) {
        fileCacheUrls.push(saveOneFileCache(currentIndex, currentCache));
        currentContentTotalSize = 0;
        currentIndex++;
        currentCache = {};
      }
      currentCache[key] = content;
      currentContentTotalSize += key.length + content.length;
    }
    if (currentContentTotalSize > 0) {
      fileCacheUrls.push(saveOneFileCache(currentIndex, currentCache));
    }
    return fileCacheUrls;
  }

  function saveOneFileCache(index: number, fileCache: Test.FileCache) {
    fs.writeFileSync(path.join(TEST_ROOT, `./testdata-file-cache-${index}.json`), JSON.stringify(fileCache));
    return path.join(TEST_DATA_BASE, `./testdata-file-cache-${index}.json`);
  }

  function saveConfig(config: Test.Config) {
    fs.writeJSONSync(path.join(TEST_ROOT, './testdata-config.json'), config);
  }

  function getBrowserNameFromEnv(env: TestRunnerCliArgs['env']) {
    switch (env) {
      case 'chrome':
        return 'ChromeTest';
      case 'chromecanary':
        return 'ChromeCanaryTest';
      case 'edge':
        return 'EdgeTest';
      case 'firefox':
        return 'FirefoxTest';
      case 'electron':
        return 'Electron';
      case 'safari':
        return 'Safari';
      case 'bs':
        return process.env.ORT_WEB_TEST_BS_BROWSERS!;
      default:
        throw new Error(`env "${env}" not supported.`);
    }
  }
}

void main();