From fed08e070a508bf29d11a44cf3b89ecd019f015d Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 6 Jul 2023 06:58:48 +0800 Subject: [PATCH] Add compiler cache in linux wasm build (#16579) ### Description Add compiler cache in wasm build to accelerate web ci ### Motivation and Context It could reduce the pipeline duration by 30 minutes. web ci could be completed in 2 hours with cache. https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=1053219&view=results --- .../templates/build-linux-wasm-step.yml | 74 ++++++++++ .../templates/linux-wasm-ci.yml | 134 ++++++++++++------ .../azure-pipelines/templates/web-ci.yml | 8 ++ .../azure-pipelines/templates/win-wasm-ci.yml | 9 ++ .../azure-pipelines/web-ci-pipeline.yml | 1 + 5 files changed, 186 insertions(+), 40 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/build-linux-wasm-step.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/build-linux-wasm-step.yml b/tools/ci_build/github/azure-pipelines/templates/build-linux-wasm-step.yml new file mode 100644 index 0000000000..82a86e2ec8 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/build-linux-wasm-step.yml @@ -0,0 +1,74 @@ +parameters: +- name: WithCache + displayName: Build with Cache + type: boolean + default: false + +# if WithCahe is false, the following parameters are not used. +- name: Today + type: string + default: "" + +- name: AdditionalKey + type: string + default: "" + +- name: CacheDir + type: string + default: "" + +- name: Arguments + type: string + +- name: DisplayName + type: string + +steps: + - task: Cache@2 + inputs: + ${{if eq(variables['Build.SourceBranchName'], 'merge')}}: + key: ' "${{parameters.TODAY}}" | ${{parameters.AdditionalKey}} | merge ' + ${{else}}: + key: '"${{parameters.TODAY}}" | ${{parameters.AdditionalKey}} | $(Build.SourceVersion) ' + path: ${{parameters.CacheDir}} + restoreKeys: | + "${{parameters.TODAY}}" | ${{parameters.AdditionalKey}} + displayName: Cache Task + condition: eq('${{parameters.WithCache}}', true) + + - ${{if eq(parameters.WithCache, true)}}: + - script: | + pushd '$(Build.SourcesDirectory)/cmake/external/emsdk' + source ./emsdk_env.sh + export PATH=$(Build.SourcesDirectory)/cmake/external/emsdk/:$PATH + export PATH=$(Build.SourcesDirectory)/cmake/external/emsdk/ccache/git-emscripten_64bit/bin:$PATH + echo $PATH + ccache -s + ccache -z + export EM_CONFIG="$(Build.SourcesDirectory)/cmake/external/emsdk/.emscripten" + + popd + pushd '$(Build.BinariesDirectory)' + python3 '$(Build.SourcesDirectory)/tools/ci_build/build.py' ${{parameters.Arguments}} + ret=$? + if [ $ret -ne 0 ]; then + echo "Build failed with error code $ret" + exit 1 + fi + ccache -s + popd + displayName: ${{parameters.DisplayName}} + env: + CCACHE_SLOPPINESS: include_file_ctime,include_file_mtime,time_macros + CCACHE_DIR: ${{parameters.CacheDir}} + _EMCC_CCACHE: 1 + EM_COMPILER_WRAPPER: ccache + EM_DIR: '$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten' + + - ${{if eq(parameters.WithCache, false)}}: + - task: PythonScript@0 + displayName: '${{parameters.DisplayName}}' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' + arguments: ${{parameters.Arguments}} + workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml index f764dc49d5..7050a346b7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-wasm-ci.yml @@ -26,6 +26,15 @@ parameters: type: boolean default: false +# In fact, it's only used on Linux for compiler cache. +- name: BuildStaticLib + type: boolean + default: false + +- name: WithCache + type: boolean + default: false + jobs: - job: build_WASM pool: @@ -35,10 +44,15 @@ jobs: buildArch: x64 CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --build_wasm --use_xnnpack --emsdk_version releases-7c905cfc1ca6699f6ccb288ae174902cfbdcf0a2-64bit ${{ parameters.ExtraBuildArgs }}' runCodesignValidationInjection: false + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache timeoutInMinutes: ${{ parameters.TimeoutInMinutes }} workspace: clean: all steps: + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() - checkout: self - task: DownloadPipelineArtifact@2 inputs: @@ -75,44 +89,87 @@ jobs: arguments: --new_dir $(Build.BinariesDirectory)/deps workingDirectory: $(Build.BinariesDirectory) - - task: PythonScript@0 - displayName: 'Build and test (node)' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm' - workingDirectory: '$(Build.BinariesDirectory)' - - task: PythonScript@0 - displayName: 'Build and test (node) (threads)' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_threads --enable_wasm_threads' - workingDirectory: '$(Build.BinariesDirectory)' - - task: PythonScript@0 - displayName: 'Build and test (browser) (simd + threads)' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads --enable_wasm_simd --enable_wasm_threads --wasm_run_tests_in_browser' - workingDirectory: '$(Build.BinariesDirectory)' - - task: PythonScript@0 - displayName: 'Build and test (node) (simd)' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd --enable_wasm_simd' - workingDirectory: '$(Build.BinariesDirectory)' + - script: | + set -ex + cd '$(Build.SourcesDirectory)/cmake/external/emsdk' + ./emsdk install 3.1.37 ccache-git-emscripten-64bit + ./emsdk activate 3.1.37 ccache-git-emscripten-64bit + ln -s $(Build.SourcesDirectory)/cmake/external/emsdk/ccache/git-emscripten_64bit/bin/ccache /usr/local/bin/ccache + displayName: 'emsdk install and activate ccache for emscripten' + condition: eq('${{ parameters.WithCache }}', 'true') + + - template: build-linux-wasm-step.yml + parameters: + Today: $(Today) + ${{ if eq(parameters.BuildStaticLib, true)}}: + AdditionalKey: wasm | ${{ parameters.BuildConfig }} | static + ${{ else }}: + AdditionalKey: wasm | ${{ parameters.BuildConfig }} + CacheDir: $(ORT_CACHE_DIR)/wasm + Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm' + DisplayName: 'Build and test (node)' + WithCache: ${{ parameters.WithCache }} + + - template: build-linux-wasm-step.yml + parameters: + Today: $(Today) + ${{ if eq(parameters.BuildStaticLib, true)}}: + AdditionalKey: wasm_threads | ${{ parameters.BuildConfig }} | static + ${{ else }}: + AdditionalKey: wasm_threads | ${{ parameters.BuildConfig }} + CacheDir: $(ORT_CACHE_DIR)/wasm_threads + Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_threads --enable_wasm_threads' + DisplayName: 'Build and test (node) (threads)' + WithCache: ${{ parameters.WithCache }} + + - template: build-linux-wasm-step.yml + parameters: + Today: $(Today) + ${{ if eq(parameters.BuildStaticLib, true)}}: + AdditionalKey: wasm_simd_threads | ${{ parameters.BuildConfig }} | static + ${{ else }}: + AdditionalKey: wasm_simd_threads | ${{ parameters.BuildConfig }} + CacheDir: $(ORT_CACHE_DIR)/wasm_simd_threads + Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads --enable_wasm_simd --enable_wasm_threads --wasm_run_tests_in_browser' + DisplayName: 'Build and test (node) (simd + threads)' + WithCache: ${{ parameters.WithCache }} + + - template: build-linux-wasm-step.yml + parameters: + Today: $(Today) + ${{ if eq(parameters.BuildStaticLib, true)}}: + AdditionalKey: wasm_simd | ${{ parameters.BuildConfig }} | static + ${{ else }}: + AdditionalKey: wasm_simd | ${{ parameters.BuildConfig }} + CacheDir: $(ORT_CACHE_DIR)/wasm_simd + Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd --enable_wasm_simd' + DisplayName: 'Build and test (node) (simd)' + WithCache: ${{ parameters.WithCache }} + - ${{ if eq(parameters.BuildJsep, true) }}: - - task: PythonScript@0 - displayName: 'Build (simd + JSEP)' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_jsep --enable_wasm_simd --use_jsep --target onnxruntime_webassembly --skip_tests' - workingDirectory: '$(Build.BinariesDirectory)' - - ${{ if eq(parameters.BuildJsep, true) }}: - - task: PythonScript@0 - displayName: 'Build (simd + threads + JSEP)' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/build.py' - arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads_jsep --enable_wasm_simd --enable_wasm_threads --use_jsep --target onnxruntime_webassembly --skip_tests' - workingDirectory: '$(Build.BinariesDirectory)' + - template: build-linux-wasm-step.yml + parameters: + Today: $(Today) + ${{ if eq(parameters.BuildStaticLib, true)}}: + AdditionalKey: wasm_simd_jsep | ${{ parameters.BuildConfig }} | static + ${{ else }}: + AdditionalKey: wasm_simd_jsep | ${{ parameters.BuildConfig }} + CacheDir: $(ORT_CACHE_DIR)/wasm_simd_jsep + Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_jsep --enable_wasm_simd --use_jsep --target onnxruntime_webassembly --skip_tests' + DisplayName: 'Build (simd + JSEP)' + WithCache: ${{ parameters.WithCache }} + - template: build-linux-wasm-step.yml + parameters: + Today: $(Today) + ${{ if eq(parameters.BuildStaticLib, true)}}: + AdditionalKey: wasm_simd_threads_jsep | ${{ parameters.BuildConfig }} | static + ${{ else }}: + AdditionalKey: wasm_simd_threads_jsep | ${{ parameters.BuildConfig }} + CacheDir: $(ORT_CACHE_DIR)/wasm_simd_threads_jsep + Arguments: '$(CommonBuildArgs) --build_dir $(Build.BinariesDirectory)/wasm_simd_threads_jsep --enable_wasm_simd --enable_wasm_threads --use_jsep --target onnxruntime_webassembly --skip_tests' + DisplayName: 'Build (simd + threads + JSEP)' + WithCache: ${{ parameters.WithCache }} + - ${{ if eq(parameters.SkipPublish, false) }}: - script: | cp $(Build.BinariesDirectory)/wasm/${{ parameters.BuildConfig }}/ort-wasm*.* $(Build.ArtifactStagingDirectory) @@ -145,6 +202,3 @@ jobs: - template: component-governance-component-detection-steps.yml parameters : condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() diff --git a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml index 8bb5860039..0212bd0b46 100644 --- a/tools/ci_build/github/azure-pipelines/templates/web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/web-ci.yml @@ -42,6 +42,10 @@ parameters: displayName: 'Extra build command line arguments' type: string +- name: WithCache + displayName: Build with Cache + type: boolean + default: false stages: - stage: Extract_commit @@ -79,6 +83,7 @@ stages: ExtraBuildArgs: '--use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}' PoolName: ${{ parameters.PoolName }} BuildJsep: ${{ parameters.BuildJsep }} + WithCache: ${{ parameters.WithCache }} - stage: Build_web_Debug dependsOn: Build_wasm_Debug @@ -104,6 +109,7 @@ stages: ExtraBuildArgs: '--target onnxruntime_webassembly --skip_tests --enable_wasm_api_exception_catching --disable_rtti --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}' PoolName: ${{ parameters.PoolName }} BuildJsep: ${{ parameters.BuildJsep }} + WithCache: ${{ parameters.WithCache }} - ${{ if eq(parameters.BuildStaticLib, 'true') }}: - stage: Build_wasm_Release_static_library @@ -117,6 +123,8 @@ stages: PoolName: ${{ parameters.PoolName }} SkipPublish: true TimeoutInMinutes: 270 + BuildStaticLib: true + WithCache: ${{ parameters.WithCache }} - stage: Build_web_Release dependsOn: Build_wasm_Release diff --git a/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml index c01d7ea67d..922f71614e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml @@ -26,6 +26,15 @@ parameters: type: boolean default: false +# In fact, it's only used on Linux for compiler cache. +- name: BuildStaticLib + type: boolean + default: false + +- name: WithCache + type: boolean + default: false + jobs: - job: build_WASM pool: ${{ parameters.PoolName }} diff --git a/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml index ae5d94c6ce..0839a9f900 100644 --- a/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml @@ -35,3 +35,4 @@ stages: WebTemplate: win-web-ci.yml UseWebPoolName: true WebPoolName: 'onnxruntime-Win-CPU-2022-web' + WithCache: true