Add compiler cache in linux wasm build (#16579)

### Description
Add compiler cache in wasm build to accelerate web ci

### Motivation and Context
It could reduce the pipeline duration by 30 minutes.
web ci could be completed in 2 hours with cache.

https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=1053219&view=results

tools/ci_build/github/azure-pipelines/templates/build-linux-wasm-step.yml

@@ -0,0 +1,74 @@
+parameters:
+- name: WithCache
+  displayName: Build with Cache
+  type: boolean
+  default: false
+
+# if WithCahe is false, the following parameters are not used.
+- name: Today
+  type: string
+  default: ""
+
+- name: AdditionalKey
+  type: string
+  default: ""
+
+- name: CacheDir
+  type: string
+  default: ""
+
+- name: Arguments
+  type: string
+
+- name: DisplayName
+  type: string
+
+steps:
+  - task: Cache@2
+    inputs:
+      ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
+        key: ' "${{parameters.TODAY}}" |  ${{parameters.AdditionalKey}}  | merge '
+      ${{else}}:
+        key: '"${{parameters.TODAY}}" |  ${{parameters.AdditionalKey}}  | $(Build.SourceVersion) '
+      path: ${{parameters.CacheDir}}
+      restoreKeys: |
+        "${{parameters.TODAY}}" |  ${{parameters.AdditionalKey}}
+    displayName: Cache Task
+    condition: eq('${{parameters.WithCache}}', true)
+
+  - ${{if eq(parameters.WithCache, true)}}:
+    - script: |
+        pushd '$(Build.SourcesDirectory)/cmake/external/emsdk'
+        source ./emsdk_env.sh
+        export PATH=$(Build.SourcesDirectory)/cmake/external/emsdk/:$PATH
+        export PATH=$(Build.SourcesDirectory)/cmake/external/emsdk/ccache/git-emscripten_64bit/bin:$PATH
+        echo $PATH
+        ccache -s
+        ccache -z
+        export EM_CONFIG="$(Build.SourcesDirectory)/cmake/external/emsdk/.emscripten"
+
+        popd
+        pushd '$(Build.BinariesDirectory)'
+        python3 '$(Build.SourcesDirectory)/tools/ci_build/build.py' ${{parameters.Arguments}}
+        ret=$?
+        if [ $ret -ne 0 ]; then
+            echo "Build failed with error code $ret"
+            exit 1
+        fi
+        ccache -s
+        popd
+      displayName: ${{parameters.DisplayName}}
+      env:
+        CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime,time_macros
+        CCACHE_DIR: ${{parameters.CacheDir}}
+        _EMCC_CCACHE: 1
+        EM_COMPILER_WRAPPER: ccache
+        EM_DIR: '$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten'
+
+  - ${{if eq(parameters.WithCache, false)}}:
+    - task: PythonScript@0
+      displayName: '${{parameters.DisplayName}}'
+      inputs:
+        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
+        arguments: ${{parameters.Arguments}}
+        workingDirectory: '$(Build.BinariesDirectory)'

tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml

@@ -26,6 +26,15 @@ parameters:
   type: boolean
   default: false
 
+# In fact, it's only used on Linux for compiler cache.
+- name: BuildStaticLib
+  type: boolean
+  default: false
+
+- name: WithCache
+  type: boolean
+  default: false
+
 jobs:
 - job: build_WASM
   pool:
@@ -35,10 +44,15 @@ jobs:
     buildArch: x64
     CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --build_wasm --use_xnnpack --emsdk_version releases-7c905cfc1ca6699f6ccb288ae174902cfbdcf0a2-64bit ${{ parameters.ExtraBuildArgs }}'
     runCodesignValidationInjection: false
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
+    ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
   timeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
   workspace:
     clean: all
   steps:
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
   - checkout: self
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -75,44 +89,87 @@ jobs:
       arguments: --new_dir $(Build.BinariesDirectory)/deps
       workingDirectory: $(Build.BinariesDirectory)
 
-  - task: PythonScript@0
-    displayName: 'Build and test (node)'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
-      arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm'
-      workingDirectory: '$(Build.BinariesDirectory)'
-  - task: PythonScript@0
-    displayName: 'Build and test (node) (threads)'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
-      arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_threads --enable_wasm_threads'
-      workingDirectory: '$(Build.BinariesDirectory)'
-  - task: PythonScript@0
-    displayName: 'Build and test (browser) (simd + threads)'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
-      arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads --enable_wasm_simd --enable_wasm_threads --wasm_run_tests_in_browser'
-      workingDirectory: '$(Build.BinariesDirectory)'
-  - task: PythonScript@0
-    displayName: 'Build and test (node) (simd)'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
-      arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd --enable_wasm_simd'
-      workingDirectory: '$(Build.BinariesDirectory)'
+  - script: |
+      set -ex
+      cd '$(Build.SourcesDirectory)/cmake/external/emsdk'
+      ./emsdk install 3.1.37 ccache-git-emscripten-64bit
+      ./emsdk activate 3.1.37 ccache-git-emscripten-64bit
+      ln -s $(Build.SourcesDirectory)/cmake/external/emsdk/ccache/git-emscripten_64bit/bin/ccache /usr/local/bin/ccache
+    displayName: 'emsdk install and activate ccache for emscripten'
+    condition: eq('${{ parameters.WithCache }}', 'true')
+
+  - template: build-linux-wasm-step.yml
+    parameters:
+      Today: $(Today)
+      ${{ if eq(parameters.BuildStaticLib, true)}}:
+        AdditionalKey: wasm | ${{ parameters.BuildConfig }} | static
+      ${{ else }}:
+        AdditionalKey: wasm | ${{ parameters.BuildConfig }}
+      CacheDir: $(ORT_CACHE_DIR)/wasm
+      Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm'
+      DisplayName: 'Build and test (node)'
+      WithCache: ${{ parameters.WithCache }}
+
+  - template: build-linux-wasm-step.yml
+    parameters:
+      Today: $(Today)
+      ${{ if eq(parameters.BuildStaticLib, true)}}:
+        AdditionalKey: wasm_threads | ${{ parameters.BuildConfig }} | static
+      ${{ else }}:
+        AdditionalKey: wasm_threads | ${{ parameters.BuildConfig }}
+      CacheDir: $(ORT_CACHE_DIR)/wasm_threads
+      Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_threads --enable_wasm_threads'
+      DisplayName: 'Build and test (node) (threads)'
+      WithCache: ${{ parameters.WithCache }}
+
+  - template: build-linux-wasm-step.yml
+    parameters:
+      Today: $(Today)
+      ${{ if eq(parameters.BuildStaticLib, true)}}:
+        AdditionalKey: wasm_simd_threads | ${{ parameters.BuildConfig }} | static
+      ${{ else }}:
+        AdditionalKey: wasm_simd_threads | ${{ parameters.BuildConfig }}
+      CacheDir: $(ORT_CACHE_DIR)/wasm_simd_threads
+      Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads --enable_wasm_simd --enable_wasm_threads --wasm_run_tests_in_browser'
+      DisplayName: 'Build and test (node) (simd + threads)'
+      WithCache: ${{ parameters.WithCache }}
+
+  - template: build-linux-wasm-step.yml
+    parameters:
+      Today: $(Today)
+      ${{ if eq(parameters.BuildStaticLib, true)}}:
+        AdditionalKey: wasm_simd | ${{ parameters.BuildConfig }} | static
+      ${{ else }}:
+        AdditionalKey: wasm_simd | ${{ parameters.BuildConfig }}
+      CacheDir: $(ORT_CACHE_DIR)/wasm_simd
+      Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd --enable_wasm_simd'
+      DisplayName: 'Build and test (node) (simd)'
+      WithCache: ${{ parameters.WithCache }}
+
   - ${{ if eq(parameters.BuildJsep, true) }}:
-    - task: PythonScript@0
-      displayName: 'Build (simd + JSEP)'
-      inputs:
-        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
-        arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_jsep --enable_wasm_simd --use_jsep --target onnxruntime_webassembly --skip_tests'
-        workingDirectory: '$(Build.BinariesDirectory)'
-  - ${{ if eq(parameters.BuildJsep, true) }}:
-    - task: PythonScript@0
-      displayName: 'Build (simd + threads + JSEP)'
-      inputs:
-        scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py'
-        arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads_jsep --enable_wasm_simd --enable_wasm_threads --use_jsep --target onnxruntime_webassembly --skip_tests'
-        workingDirectory: '$(Build.BinariesDirectory)'
+    - template: build-linux-wasm-step.yml
+      parameters:
+        Today: $(Today)
+        ${{ if eq(parameters.BuildStaticLib, true)}}:
+          AdditionalKey: wasm_simd_jsep | ${{ parameters.BuildConfig }} | static
+        ${{ else }}:
+          AdditionalKey: wasm_simd_jsep | ${{ parameters.BuildConfig }}
+        CacheDir: $(ORT_CACHE_DIR)/wasm_simd_jsep
+        Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_jsep --enable_wasm_simd --use_jsep --target onnxruntime_webassembly --skip_tests'
+        DisplayName: 'Build (simd + JSEP)'
+        WithCache: ${{ parameters.WithCache }}
+    - template: build-linux-wasm-step.yml
+      parameters:
+        Today: $(Today)
+        ${{ if eq(parameters.BuildStaticLib, true)}}:
+          AdditionalKey: wasm_simd_threads_jsep | ${{ parameters.BuildConfig }} | static
+        ${{ else }}:
+          AdditionalKey: wasm_simd_threads_jsep | ${{ parameters.BuildConfig }}
+        CacheDir: $(ORT_CACHE_DIR)/wasm_simd_threads_jsep
+        Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads_jsep --enable_wasm_simd --enable_wasm_threads --use_jsep --target onnxruntime_webassembly --skip_tests'
+        DisplayName: 'Build (simd + threads + JSEP)'
+        WithCache: ${{ parameters.WithCache }}
+
   - ${{ if eq(parameters.SkipPublish, false) }}:
     - script: |
         cp $(Build.BinariesDirectory)/wasm/${{ parameters.BuildConfig }}/ort-wasm*.* $(Build.ArtifactStagingDirectory)
@@ -145,6 +202,3 @@ jobs:
   - template: component-governance-component-detection-steps.yml
     parameters :
       condition : 'succeeded'
-  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
-    displayName: 'Clean Agent Directories'
-    condition: always()

tools/ci_build/github/azure-pipelines/templates/web-ci.yml

@@ -42,6 +42,10 @@ parameters:
   displayName: 'Extra build command line arguments'
   type: string
 
+- name: WithCache
+  displayName: Build with Cache
+  type: boolean
+  default: false
 
 stages:
 - stage: Extract_commit
@@ -79,6 +83,7 @@ stages:
       ExtraBuildArgs: '--use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}'
       PoolName: ${{ parameters.PoolName }}
       BuildJsep: ${{ parameters.BuildJsep }}
+      WithCache: ${{ parameters.WithCache }}
 
 - stage: Build_web_Debug
   dependsOn: Build_wasm_Debug
@@ -104,6 +109,7 @@ stages:
       ExtraBuildArgs: '--target onnxruntime_webassembly --skip_tests --enable_wasm_api_exception_catching --disable_rtti --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}'
       PoolName: ${{ parameters.PoolName }}
       BuildJsep: ${{ parameters.BuildJsep }}
+      WithCache: ${{ parameters.WithCache }}
 
 - ${{ if eq(parameters.BuildStaticLib, 'true') }}:
   - stage: Build_wasm_Release_static_library
@@ -117,6 +123,8 @@ stages:
         PoolName: ${{ parameters.PoolName }}
         SkipPublish: true
         TimeoutInMinutes: 270
+        BuildStaticLib: true
+        WithCache: ${{ parameters.WithCache }}
 
 - stage: Build_web_Release
   dependsOn: Build_wasm_Release

tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml

@@ -26,6 +26,15 @@ parameters:
   type: boolean
   default: false
 
+# In fact, it's only used on Linux for compiler cache.
+- name: BuildStaticLib
+  type: boolean
+  default: false
+
+- name: WithCache
+  type: boolean
+  default: false
+
 jobs:
 - job: build_WASM
   pool: ${{ parameters.PoolName }}

tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml

@@ -35,3 +35,4 @@ stages:
     WebTemplate: win-web-ci.yml
     UseWebPoolName: true
     WebPoolName: 'onnxruntime-Win-CPU-2022-web'
+    WithCache: true