From 850baced337365913a2fc8c3f88edc94014a1d45 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:43:42 -0700 Subject: [PATCH] [web] a few updates to web pipeline (#17485) ### Description Update the Web CI pipelines: - remove parameter 'WebTemplate': Since we start to support webgpu, the linux-web-ci.yml is no longer working and it is already out-of-date. remove this file and parameter so that we always use win-web-ci.yml - change flag `RunWebGpuTests` into 2 flags, for release and debug. Currently for CI we only run webgpu tests on release build. But we want to have the capability to run webgpu tests on debug build as well. After this PR is merged, next step is to enable both Debug and Release webgpu tests in PostMerge pipeline. --- .../templates/linux-web-ci.yml | 147 ------------------ .../azure-pipelines/templates/web-ci.yml | 22 +-- .../azure-pipelines/web-ci-pipeline.yml | 4 +- 3 files changed, 15 insertions(+), 158 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/templates/linux-web-ci.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-web-ci.yml deleted file mode 100644 index d2f5df1d17..0000000000 --- a/tools/ci_build/github/azure-pipelines/templates/linux-web-ci.yml +++ /dev/null @@ -1,147 +0,0 @@ -parameters: -- name: CommitOverride - type: boolean - default: false - -- name: BuildConfig - type: string - default: 'Debug' - -- name: NpmPackagingMode - type: string - default: 'dev' - -- name: PoolName - type: string - default: 'onnxruntime-Ubuntu2004-AMD-CPU' - -- name: PackageName - displayName: 'Package name' - type: string - default: 'NPM_packages' - -jobs: -- job: build_onnxruntime_web - pool: ${{ parameters.PoolName }} - - variables: - runCodesignValidationInjection: false - timeoutInMinutes: 60 - workspace: - clean: all - steps: - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - - checkout: self - submodules: false - - task: DownloadPipelineArtifact@2 - inputs: - artifact: '__commit' - path: $(Pipeline.Workspace) - displayName: 'Get commit SHA' - condition: eq('${{ parameters.CommitOverride }}', 'true') - - script: | - export __commit__=<$(Pipeline.Workspace)/__commit.txt - git fetch origin +$__commit__:refs/remotes/origin/$__commit__ - git checkout --force $__commit__ - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'Read commit SHA and checkout' - condition: eq('${{ parameters.CommitOverride }}', 'true') - - script: | - echo.$(Build.SourceVersion)>$(Pipeline.Workspace)/__commit.txt - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'Write commit SHA to __commit.txt' - condition: ne('${{ parameters.CommitOverride }}', 'true') - - script: | - git submodule sync -- cmake/external/onnx - git submodule update --init -- cmake/external/onnx - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'Checkout submodule onnx' - - task: NodeTool@0 - inputs: - versionSpec: '16.x' - - task: DownloadPipelineArtifact@2 - inputs: - patterns: '${{ parameters.BuildConfig }}_*/**/*' - path: $(Pipeline.Workspace)/artifacts - displayName: 'Download WebAssembly artifacts' - - task: CopyFiles@2 - inputs: - sourceFolder: $(Pipeline.Workspace)/artifacts - contents: | - **/*.wasm - targetFolder: $(Build.SourcesDirectory)/js/web/dist - flattenFolders: true - displayName: 'Binplace dist files' - - task: CopyFiles@2 - inputs: - sourceFolder: $(Pipeline.Workspace)/artifacts - contents: | - **/*.js - targetFolder: $(Build.SourcesDirectory)/js/web/lib/wasm/binding - flattenFolders: true - displayName: 'Binplace js files' - - template: linux-web-init-and-check.yml - - task: Cache@2 - inputs: - key: onnxtestdata | $(Build.SourcesDirectory)\js\scripts\prepare-onnx-node-tests.ts - restoreKeys: | - onnxtestdata | $(Build.SourcesDirectory)\js\scripts\prepare-onnx-node-tests.ts - path: $(Build.SourcesDirectory)/js/test/ - cacheHitVar: CACHE_RESTORED - displayName: 'Cache ONNX node test data' - - task: Bash@3 - inputs: - targetType: 'inline' - script: find "$(Build.SourcesDirectory)/js/test/" -type f - condition: and(not(canceled()), eq(variables.CACHE_RESTORED, 'true')) - displayName: 'List ONNX node test data' - - task: PowerShell@2 - inputs: - filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1' - arguments: '$(NpmPackagingMode) $(Build.SourcesDirectory) web' - workingDirectory: $(Build.BinariesDirectory) - errorActionPreference: stop - displayName: 'Pack NPM packages' - - script: | - npm test - workingDirectory: '$(Build.SourcesDirectory)/js/web' - displayName: 'Run ort-web tests' - - script: | - npm test -- --webgl-texture-pack-mode -b=webgl - workingDirectory: '$(Build.SourcesDirectory)/js/web' - displayName: 'Run ort-web tests - WebGL: packed mode' - - script: | - npm test -- --wasm-enable-proxy -b=wasm - workingDirectory: '$(Build.SourcesDirectory)/js/web' - displayName: 'Run ort-web tests - WebAssembly: proxy' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - - script: | - npm run test:e2e - workingDirectory: '$(Build.SourcesDirectory)/js/web' - displayName: 'E2E package consuming test' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - - task: CopyFiles@2 - inputs: - sourceFolder: $(Build.SourcesDirectory)/js/common - contents: onnxruntime-common-*.tgz - targetFolder: $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts (onnxruntime-common)' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - - task: CopyFiles@2 - inputs: - sourceFolder: $(Build.SourcesDirectory)/js/web - contents: onnxruntime-web-*.tgz - targetFolder: $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts (onnxruntime-web)' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - - task: PublishPipelineArtifact@0 - inputs: - artifactName: '${{ parameters.PackageName }}' - targetPath: '$(Build.ArtifactStagingDirectory)' - displayName: 'Publish Pipeline Artifact' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml index e0a85cc197..0b7bd3f645 100644 --- a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml @@ -27,14 +27,14 @@ parameters: - name: WASMTemplate type: string default: win-wasm-ci.yml -- name: WebTemplate - type: string - default: win-web-ci.yml # parameter couldn't be compared by string, so add one boolean parameter. - name: UseWebPoolName type: boolean default: false -- name: RunWebGpuTests +- name: RunWebGpuTestsForReleaseBuild + type: boolean + default: false +- name: RunWebGpuTestsForDebugBuild type: boolean default: false - name: WebGpuPoolName @@ -104,16 +104,20 @@ stages: - stage: Build_web_Debug dependsOn: Build_wasm_Debug jobs: - - template: ${{ parameters.WebTemplate }} + - template: win-web-ci.yml parameters: CommitOverride: true BuildConfig: 'Debug' NpmPackagingMode: ${{ parameters.NpmPackagingMode }} ${{ if eq(parameters.UseWebPoolName, true)}}: - PoolName: ${{ parameters.WebCpuPoolName }} + ${{ if eq(parameters.RunWebGpuTestsForDebugBuild, true)}}: + PoolName: ${{ parameters.WebGpuPoolName }} + ${{ else }}: + PoolName: ${{ parameters.WebCpuPoolName }} ${{ else }}: PoolName: ${{ parameters.PoolName }} PackageName: ${{ parameters.PackageName }} + RunWebGpuTests: ${{ parameters.RunWebGpuTestsForDebugBuild }} - stage: Build_wasm_Release dependsOn: Precheck_and_extract_commit @@ -145,20 +149,20 @@ stages: - stage: Build_web_Release dependsOn: Build_wasm_Release jobs: - - template: ${{ parameters.WebTemplate }} + - template: win-web-ci.yml parameters: CommitOverride: true BuildConfig: 'Release' NpmPackagingMode: ${{ parameters.NpmPackagingMode }} ${{ if eq(parameters.UseWebPoolName, true)}}: - ${{ if eq(parameters.RunWebGpuTests, true)}}: + ${{ if eq(parameters.RunWebGpuTestsForReleaseBuild, true)}}: PoolName: ${{ parameters.WebGpuPoolName }} ${{ else }}: PoolName: ${{ parameters.WebCpuPoolName }} ${{ else }}: PoolName: ${{ parameters.PoolName }} PackageName: ${{ parameters.PackageName }} - RunWebGpuTests: ${{ parameters.RunWebGpuTests }} + RunWebGpuTests: ${{ parameters.RunWebGpuTestsForReleaseBuild }} # Disable BrowserStack test # TODO: fix and re-enable in PostMerge test diff --git a/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml index 38b4814a4c..63dabf5eab 100644 --- a/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml @@ -57,9 +57,9 @@ stages: BuildStaticLib: true ExtraBuildArgs: $(ExtraBuildArgs) WASMTemplate: linux-wasm-ci.yml - WebTemplate: win-web-ci.yml UseWebPoolName: true - RunWebGpuTests: true + RunWebGpuTestsForDebugBuild: false + RunWebGpuTestsForReleaseBuild: true WebGpuPoolName: 'onnxruntime-Win2022-webgpu-A10' WebCpuPoolName: 'onnxruntime-Win-CPU-2022-web' WithCache: true