From 0313dd1f65a329d001c32b1a6ba45bc52d0f16fa Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:16:59 -0700 Subject: [PATCH] Update Web CI to use data dir under Agent.TempDirectory (#20074) ### Description Update Web CI to use data dir under Agent.TempDirectory This change fixes the random failure caused by unstable access to karma temp directory (which is under AppData\Local\Temp) on CI pipeline --- js/web/karma.conf.js | 9 ++-- js/web/script/test-runner-cli-args.ts | 6 +++ js/web/script/test-runner-cli.ts | 5 +- .../azure-pipelines/templates/win-web-ci.yml | 50 +++++++++++++++---- .../templates/win-web-multi-browsers.yml | 21 ++++++-- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/js/web/karma.conf.js b/js/web/karma.conf.js index 9e44d9c0d9..507da0de2b 100644 --- a/js/web/karma.conf.js +++ b/js/web/karma.conf.js @@ -9,6 +9,8 @@ const karmaPlugins = args['karma-plugins'] || undefined; const timeoutMocha = args['timeout-mocha'] || 60000; const forceLocalHost = !!args['force-localhost']; +// user data directory; will be passed to the Edge/Chrome/ChromeCanary/Firefox launchers +const userDataDir = args['user-data-dir']; // parse chromium flags let chromiumFlags = args['chromium-flags']; if (!chromiumFlags) { @@ -87,9 +89,10 @@ module.exports = function(config) { listenAddress, customLaunchers: { // Chromium-based browsers - EdgeTest: {base: 'Edge', flags: chromiumFlags}, - ChromeTest: {base: 'Chrome', flags: chromiumFlags}, - ChromeCanaryTest: {base: 'ChromeCanary', flags: chromiumFlags}, + EdgeTest: {base: 'Edge', flags: chromiumFlags, edgeDataDir: userDataDir}, + ChromeTest: {base: 'Chrome', flags: chromiumFlags, chromeDataDir: userDataDir}, + ChromeCanaryTest: {base: 'ChromeCanary', flags: chromiumFlags, chromeDataDir: userDataDir}, + FirefoxTest: {base: 'Firefox', profile: userDataDir}, // // ==== BrowserStack browsers ==== diff --git a/js/web/script/test-runner-cli-args.ts b/js/web/script/test-runner-cli-args.ts index b2b212bdb9..745f504b04 100644 --- a/js/web/script/test-runner-cli-args.ts +++ b/js/web/script/test-runner-cli-args.ts @@ -103,6 +103,7 @@ Options: --no-sandbox This flag will be passed to Chrome. Sometimes Chrome need this flag to work together with Karma. + --user-data-dir=<...> This flag will be passed to browsers to specify the user data directory. --chromium-flags=<...> This flag will be passed to Chrome and Edge browsers. Can be used multiple times. Examples: @@ -195,6 +196,7 @@ export interface TestRunnerCliArgs { webnnOptions?: InferenceSession.WebNNExecutionProviderOption; globalEnvFlags?: Test.Options['globalEnvFlags']; noSandbox?: boolean; + userDataDir?: string; chromiumFlags: string[]; } @@ -477,6 +479,9 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs // Option: --no-sandbox const noSandbox = !!args['no-sandbox']; + // Option: --user-data-dir + const userDataDir = args['user-data-dir']; + // parse chromium flags let chromiumFlags = args['chromium-flags']; if (!chromiumFlags) { @@ -515,6 +520,7 @@ export function parseTestRunnerCliArgs(cmdlineArgs: string[]): TestRunnerCliArgs wasmOptions, globalEnvFlags, noSandbox, + userDataDir, chromiumFlags }; } diff --git a/js/web/script/test-runner-cli.ts b/js/web/script/test-runner-cli.ts index ace64e9532..03d637b35b 100644 --- a/js/web/script/test-runner-cli.ts +++ b/js/web/script/test-runner-cli.ts @@ -573,6 +573,9 @@ async function main() { karmaArgs.push('--log-level debug'); } karmaArgs.push(`--bundle-mode=${args.bundleMode}`); + if (args.userDataDir) { + karmaArgs.push(`--user-data-dir="${args.userDataDir}"`); + } karmaArgs.push(...chromiumFlags.map(flag => `--chromium-flags=${flag}`)); if (browser.startsWith('Edge')) { // There are currently 2 Edge browser launchers: @@ -671,7 +674,7 @@ async function main() { case 'edge': return 'EdgeTest'; case 'firefox': - return 'Firefox'; + return 'FirefoxTest'; case 'electron': return 'Electron'; case 'safari': diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml index 1eb2ee6f64..b7aee559cf 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml @@ -153,31 +153,61 @@ jobs: errorActionPreference: stop displayName: 'Pack NPM packages' - script: | - npm test -- -e=chrome -b=webgl,wasm --karma-debug + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\01 + npm test -- -e=chrome -b=webgl,wasm --user-data-dir=$(Agent.TempDirectory)\web\test\01 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (wasm,webgl backend)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'false') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'false')) - script: | - npm test -- -e=chrome -b=webgl,wasm,webgpu --karma-debug $(webgpuCommandlineExtraFlags) + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\02 + npm test -- -e=chrome -b=webgl,wasm,webgpu $(webgpuCommandlineExtraFlags) --user-data-dir=$(Agent.TempDirectory)\web\test\02 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (ALL backends)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'true') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'true')) - script: | - npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-tensor --karma-debug $(webgpuCommandlineExtraFlags) + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\03 + npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-tensor $(webgpuCommandlineExtraFlags) --user-data-dir=$(Agent.TempDirectory)\web\test\03 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (Suite1, webgpu, IO-binding=gpu-tensor)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'true') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'true')) - script: | - npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-location --karma-debug $(webgpuCommandlineExtraFlags) + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\04 + npm test -- suite1 -e=chrome -b=webgpu --io-binding=gpu-location $(webgpuCommandlineExtraFlags) --user-data-dir=$(Agent.TempDirectory)\web\test\04 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests (Suite1, webgpu, IO-binding=gpu-location)' - condition: eq('${{ parameters.RunWebGpuTests }}', 'true') + condition: and(succeeded(), eq('${{ parameters.RunWebGpuTests }}', 'true')) - script: | - npm test -- --webgl.pack -b=webgl -e=chrome --karma-debug + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\05 + npm test -- --webgl.pack -b=webgl -e=chrome --user-data-dir=$(Agent.TempDirectory)\web\test\05 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebGL: packed mode' - script: | - npm test -- --wasm.proxy -b=wasm -e=chrome --karma-debug + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test\06 + npm test -- --wasm.proxy -b=wasm -e=chrome --user-data-dir=$(Agent.TempDirectory)\web\test\06 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebAssembly: proxy' condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml index 79bf0b5e71..00109b348e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-multi-browsers.yml @@ -68,15 +68,30 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm ci /js/web/' - script: | - npm test -- suite0 -b=wasm,webgl --wasm.initTimeout=30000 --file-cache + powershell "Get-WmiObject Win32_Process -Filter \"name = 'chrome.exe'\" | Format-List CommandLine" + displayName: 'Check active Chrome processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test_multi_browsers\01 + npm test -- suite0 -e=chrome -b=wasm,webgl --wasm.initTimeout=30000 --file-cache --user-data-dir=$(Agent.TempDirectory)\web\test_multi_browsers\01 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Chrome)' - script: | - npm test -- suite0 -b=wasm,webgl -e=firefox --wasm.initTimeout=30000 --file-cache + powershell "Get-WmiObject Win32_Process -Filter \"name = 'firefox.exe'\" | Format-List CommandLine" + displayName: 'Check active Firefox processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test_multi_browsers\02 + npm test -- suite0 -b=wasm,webgl -e=firefox --wasm.initTimeout=30000 --file-cache --user-data-dir=$(Agent.TempDirectory)\web\test_multi_browsers\02 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Firefox)' - script: | - npm test -- suite0 -b=wasm,webgl -e=edge --wasm.initTimeout=30000 --file-cache + powershell "Get-WmiObject Win32_Process -Filter \"name = 'msedge.exe'\" | Format-List CommandLine" + displayName: 'Check active Edge processes (before test)' + condition: and(succeeded(), eq('$(Agent.Diagnostic)', 'true')) + - script: | + mkdir $(Agent.TempDirectory)\web\test_multi_browsers\03 + npm test -- suite0 -b=wasm,webgl -e=edge --wasm.initTimeout=30000 --file-cache --user-data-dir=$(Agent.TempDirectory)\web\test_multi_browsers\03 workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'npm test (Suite0, Edge)' - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3