diff --git a/tools/ci_build/github/azure-pipelines/templates/clean-agent-build-directory-step.yml b/tools/ci_build/github/azure-pipelines/templates/clean-agent-build-directory-step.yml
index 310b89c8e9..efef0c79ac 100644
--- a/tools/ci_build/github/azure-pipelines/templates/clean-agent-build-directory-step.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/clean-agent-build-directory-step.yml
@@ -7,3 +7,4 @@ steps:
 - script: rm -rf $(Agent.BuildDirectory)
   displayName: Clean build files (POSIX)
   condition: not(eq(variables['Agent.OS'], 'Windows_NT')) # and always()
+  continueOnError: true  # continuing on error for this step, since linux build folder is somehow getting permission issue