[js/web] allow unittest (onnxruntime_test_all) to run in browser (#14820)

### Description
allow onnxruntime_test_all to run in browser for WebAssembly build (use
flag `--wasm_run_tests_in_browser`).

To output the logs from stdout correctly, this test needs to be build
with `--enable_wasm_threads`.

cmake/CMakeLists.txt

@@ -158,6 +158,7 @@ option(onnxruntime_ENABLE_WEBASSEMBLY_THREADS "Enable this option to create WebA
 option(onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING "Enable this option to turn on exception catching" OFF)
 option(onnxruntime_ENABLE_WEBASSEMBLY_API_EXCEPTION_CATCHING "Enable this option to turn on api exception catching" OFF)
 option(onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING "Enable this option to turn on exception throwing even if the build disabled exceptions support" OFF)
+option(onnxruntime_WEBASSEMBLY_RUN_TESTS_IN_BROWSER "Enable this option to run tests in browser instead of Node.js" OFF)
 option(onnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO "Enable this option to turn on DWARF format debug info" OFF)
 option(onnxruntime_ENABLE_WEBASSEMBLY_PROFILING "Enable this option to turn on WebAssembly profiling and preserve function names" OFF)
 option(onnxruntime_ENABLE_WEBASSEMBLY_OUTPUT_OPTIMIZED_MODEL "Enable this option to allow WebAssembly to output optimized model" OFF)

cmake/onnxruntime_unittests.cmake

@@ -153,23 +153,60 @@ function(AddTest)
     xctest_add_test(xctest.${_UT_TARGET} ${_UT_TARGET}_xc)
   else()
     if (onnxruntime_BUILD_WEBASSEMBLY)
-      find_program(NODE_EXECUTABLE node required)
-      if (NOT NODE_EXECUTABLE)
-        message(FATAL_ERROR "Node is required for unit tests")
+      # the following code are copied from onnxruntime_nodejs.cmake
+      if (CMAKE_HOST_WIN32)
+          set(NPM_CLI npm.cmd)
+      else()
+          set(NPM_CLI npm)
       endif()
 
-      set(TEST_NODE_FLAGS)
-      if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
-        list(APPEND TEST_NODE_FLAGS "--experimental-wasm-threads")
+      # verify Node.js and NPM
+      execute_process(COMMAND node --version
+          OUTPUT_VARIABLE node_version
+          RESULT_VARIABLE had_error
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+      if(had_error)
+          message(FATAL_ERROR "Failed to find Node.js: " ${had_error})
       endif()
-      if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD)
-        list(APPEND TEST_NODE_FLAGS "--experimental-wasm-simd")
+      execute_process(COMMAND ${NPM_CLI} --version
+          OUTPUT_VARIABLE npm_version
+          RESULT_VARIABLE had_error
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+      if(had_error)
+          message(FATAL_ERROR "Failed to find NPM: " ${had_error})
       endif()
 
-      add_test(NAME ${_UT_TARGET}
-        COMMAND ${NODE_EXECUTABLE} ${TEST_NODE_FLAGS} ${_UT_TARGET}.js ${TEST_ARGS}
-        WORKING_DIRECTORY $<TARGET_FILE_DIR:${_UT_TARGET}>
-      )
+      if (onnxruntime_WEBASSEMBLY_RUN_TESTS_IN_BROWSER)
+        add_custom_command(TARGET ${_UT_TARGET} POST_BUILD
+          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TEST_SRC_DIR}/wasm/package.json $<TARGET_FILE_DIR:${_UT_TARGET}>
+          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TEST_SRC_DIR}/wasm/package-lock.json $<TARGET_FILE_DIR:${_UT_TARGET}>
+          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TEST_SRC_DIR}/wasm/karma.conf.js $<TARGET_FILE_DIR:${_UT_TARGET}>
+          COMMAND ${NPM_CLI} ci
+          WORKING_DIRECTORY $<TARGET_FILE_DIR:${_UT_TARGET}>
+        )
+
+        set(TEST_NPM_FLAGS)
+        if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
+          list(APPEND TEST_NPM_FLAGS "--wasm-threads")
+        endif()
+        add_test(NAME ${_UT_TARGET}
+          COMMAND ${NPM_CLI} test -- ${TEST_NPM_FLAGS} --entry=${_UT_TARGET} ${TEST_ARGS}
+          WORKING_DIRECTORY $<TARGET_FILE_DIR:${_UT_TARGET}>
+        )
+      else()
+        set(TEST_NODE_FLAGS)
+        if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
+          list(APPEND TEST_NODE_FLAGS "--experimental-wasm-threads")
+        endif()
+        if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD)
+          list(APPEND TEST_NODE_FLAGS "--experimental-wasm-simd")
+        endif()
+
+        add_test(NAME ${_UT_TARGET}
+          COMMAND node ${TEST_NODE_FLAGS} ${_UT_TARGET}.js ${TEST_ARGS}
+          WORKING_DIRECTORY $<TARGET_FILE_DIR:${_UT_TARGET}>
+        )
+      endif()
     else()
       add_test(NAME ${_UT_TARGET}
         COMMAND ${_UT_TARGET} ${TEST_ARGS}
@@ -797,8 +834,8 @@ if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
   target_link_libraries(onnxruntime_test_all PRIVATE Python::Python)
 endif()
 if (onnxruntime_BUILD_WEBASSEMBLY)
-  set_target_properties(onnxruntime_test_all PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/dump-test-result-in-nodejs.js)
-  set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 --pre-js \"${TEST_SRC_DIR}/wasm/dump-test-result-in-nodejs.js\" -s \"EXPORTED_RUNTIME_METHODS=['FS']\" --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s EXIT_RUNTIME=1")
+  set_target_properties(onnxruntime_test_all PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/onnxruntime_test_all_adapter.js)
+  set_target_properties(onnxruntime_test_all PROPERTIES LINK_FLAGS "-s ALLOW_MEMORY_GROWTH=1 --pre-js \"${TEST_SRC_DIR}/wasm/onnxruntime_test_all_adapter.js\" -s \"EXPORTED_RUNTIME_METHODS=['FS']\" --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s EXIT_RUNTIME=1")
   if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
     set_property(TARGET onnxruntime_test_all APPEND_STRING PROPERTY LINK_FLAGS " -s USE_PTHREADS=1 -s PROXY_TO_PTHREAD=1")
   endif()

onnxruntime/test/wasm/dump-test-result-in-nodejs.js

@@ -1,30 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-'use strict';
-
-// This file is used to be injected into "onnxruntime_test_all" as specified by flag "--pre-js" by emcc.
-// It dumps the test report file from emscripten's MEMFS to real file system
-
-// Module is predefined for scripts injected from "--pre-js"
-(function () {
-    if (typeof process !== 'undefined') {
-        // check for flag "--gtest_output=xml:"
-        const argGtestOutputPrefix = '--gtest_output=xml:';
-
-        for (const arg of process.argv) {
-            if (arg.startsWith(argGtestOutputPrefix)) {
-                const filename = arg.substring(argGtestOutputPrefix.length);
-                if (filename) {
-                    Module["onExit"] = function () {
-                        // read test output from MEMFS and write to real file system.
-                        const filedata = Module.FS.readFile(filename);
-                        require('fs').writeFileSync(filename, filedata);
-                    };
-                }
-                break;
-            }
-        }
-    }
-})();
-

onnxruntime/test/wasm/karma.conf.js

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+const THREADS = process.argv.includes('--wasm-threads');
+
+const files = [];
+const proxies = {};
+const chromeFlags = ['--no-sandbox'];
+for (const arg of process.argv) {
+  if (arg.startsWith('--entry=')) {
+    const entry = arg.substring('--entry='.length);
+    files.push({pattern: `${entry}.js`, watched: false});
+    files.push({pattern: `${entry}.data`, included: false, watched: false, nocache: true });
+    files.push({pattern: `${entry}.wasm`, included: false});
+    proxies[`/${entry}.data`] = `/base/${entry}.data`;
+    if (THREADS) {
+      files.push({pattern: `${entry}.worker.js`, included: false, watched: false });
+      proxies[`/${entry}.worker.js`] = `/base/${entry}.worker.js`;
+      chromeFlags.push('--enable-features=SharedArrayBuffer');
+    }
+    break;
+  }
+}
+
+if (files.length === 0) {
+  console.error('No entry file specified. Use --entry= to specify the entry file.');
+  process.exit(1);
+}
+
+// gtest reporter writes the test results to the file specified by the --gtest_output flag.
+const gtestReporter = {'reporter:gtest': ['type', function() {
+  this.onBrowserComplete = function(browser, result) {
+    if (result.file) {
+      require('fs').writeFileSync(result.file, result.data);
+    }
+  };
+}]};
+
+module.exports = function(config) {
+    config.set({
+      basePath: '.',
+      files,
+      proxies,
+      mime: {
+        'application/octet-stream': ['data']
+      },
+      plugins: [
+        require('karma-chrome-launcher'),
+        require('@chiragrupani/karma-chromium-edge-launcher'),
+        gtestReporter],
+      browsers: ['ChromeTest'],
+      reporters: ['progress', 'gtest'],
+      client: {
+        captureConsole: true,
+        // Pass the gtest flags to the test runner
+        args: process.argv.filter(arg => arg.startsWith('--gtest_'))
+      },
+      browserDisconnectTimeout: 600000,
+      // allow running tests for 30 minutes
+      browserNoActivityTimeout: 30 * 60 * 1000,
+      customLaunchers: {
+        ChromeTest: {
+          base: 'ChromeCanary',
+          flags: chromeFlags
+        },
+        EdgeTest: {
+          base: 'Edge',
+          flags: chromeFlags
+        }
+      }
+    });
+  };

onnxruntime/test/wasm/onnxruntime_test_all_adapter.js

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+// This file is used to be injected into "onnxruntime_test_all" as specified by flag "--pre-js" by emcc.
+// It dumps the test report file from emscripten's MEMFS to real file system
+
+// Module is predefined for scripts injected from "--pre-js"
+(function () {
+    let args = [];
+    let onTestComplete;
+    let gtestOutputFilepath = '';
+    let gtestOutputFiledata;
+    if (typeof process !== 'undefined') {
+        // In Node.js
+        args = process.argv;
+        onTestComplete = function () {
+            if (gtestOutputFilepath) {
+                require('fs').writeFileSync(gtestOutputFilepath, gtestOutputFiledata);
+            }
+        };
+    } else if (typeof __karma__ !== 'undefined') {
+        // In browser (launched by karma)
+        args = __karma__.config.args;
+        onTestComplete = function (exitCode) {
+            __karma__.result({
+                id: '',
+                description: '',
+                suite: [],
+                success: exitCode === 0,
+                log: []
+              });
+            __karma__.complete({file: gtestOutputFilepath, data: gtestOutputFiledata});
+        };
+
+        Module["arguments"] = args;
+        __karma__.info({total: 1});
+        __karma__.start = function () {};
+    }
+
+    // check for flag "--gtest_output=xml:"
+    const argGtestOutputPrefix = '--gtest_output=xml:';
+
+    for (const arg of args) {
+        if (arg.startsWith(argGtestOutputPrefix)) {
+            gtestOutputFilepath = arg.substring(argGtestOutputPrefix.length);
+            break;
+        }
+    }
+
+    Module["onExit"] = function(exitCode) {
+        if (gtestOutputFilepath) {
+            gtestOutputFiledata = FS.readFile(gtestOutputFilepath);
+        }
+        onTestComplete(exitCode);
+    };
+
+})();

onnxruntime/test/wasm/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "wasm-test-adapter",
+  "private": true,
+  "scripts": {
+    "test": "karma start --browsers EdgeTest --single-run"
+  },
+  "devDependencies": {
+    "@chiragrupani/karma-chromium-edge-launcher": "^2.2.2",
+    "karma": "^6.4.1",
+    "karma-chrome-launcher": "^3.1.1"
+  }
+}

tools/ci_build/build.py

@@ -414,6 +414,7 @@ def parse_arguments():
         help="Enable exception throwing in WebAssembly, this will override default disabling exception throwing "
         "behavior when disable exceptions.",
     )
+    parser.add_argument("--wasm_run_tests_in_browser", action="store_true", help="Run WebAssembly tests in browser")
 
     parser.add_argument(
         "--enable_wasm_profiling", action="store_true", help="Enable WebAsselby profiling and preserve function names"
@@ -955,6 +956,7 @@ def generate_build_tree(
         + ("ON" if args.enable_wasm_api_exception_catching else "OFF"),
         "-Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING="
         + ("ON" if args.enable_wasm_exception_throwing_override else "OFF"),
+        "-Donnxruntime_WEBASSEMBLY_RUN_TESTS_IN_BROWSER=" + ("ON" if args.wasm_run_tests_in_browser else "OFF"),
         "-Donnxruntime_ENABLE_WEBASSEMBLY_THREADS=" + ("ON" if args.enable_wasm_threads else "OFF"),
         "-Donnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO=" + ("ON" if args.enable_wasm_debug_info else "OFF"),
         "-Donnxruntime_ENABLE_WEBASSEMBLY_PROFILING=" + ("ON" if args.enable_wasm_profiling else "OFF"),

tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml

@@ -73,25 +73,25 @@ jobs:
     workingDirectory: '$(Build.BinariesDirectory)'
     displayName: 'Install python modules'
   - task: PythonScript@0
-    displayName: 'Build and test'
+    displayName: 'Build and test (node)'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
       arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm'
       workingDirectory: '$(Build.BinariesDirectory)'
   - task: PythonScript@0
-    displayName: 'Build and test (threads)'
+    displayName: 'Build and test (node) (threads)'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
       arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_threads --path_to_protoc_exe $(Build.BinariesDirectory)\wasm\host_protoc\Release\protoc.exe --enable_wasm_threads'
       workingDirectory: '$(Build.BinariesDirectory)'
   - task: PythonScript@0
-    displayName: 'Build and test (simd + threads)'
+    displayName: 'Build and test (browser) (simd + threads)'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_simd_threads --path_to_protoc_exe $(Build.BinariesDirectory)\wasm\host_protoc\Release\protoc.exe --enable_wasm_simd --enable_wasm_threads'
+      arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_simd_threads --path_to_protoc_exe $(Build.BinariesDirectory)\wasm\host_protoc\Release\protoc.exe --enable_wasm_simd --enable_wasm_threads --wasm_run_tests_in_browser'
       workingDirectory: '$(Build.BinariesDirectory)'
   - task: PythonScript@0
-    displayName: 'Build and test (simd)'
+    displayName: 'Build and test (node) (simd)'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
       arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)\wasm_simd --path_to_protoc_exe $(Build.BinariesDirectory)\wasm\host_protoc\Release\protoc.exe --enable_wasm_simd'