diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
index 115bd52201..9279e7b5a0 100644
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@@ -113,6 +113,9 @@ option(onnxruntime_ENABLE_TRAINING_E2E_TESTS "Enable training end-to-end tests."
 option(onnxruntime_USE_HOROVOD "Build with HOROVOD support" OFF)
 option(onnxruntime_USE_NCCL "Build with NCCL support" ON)
 
+# A special build option only used for gathering code coverage info
+option(onnxruntime_RUN_MODELTEST_IN_DEBUG_MODE "Run model tests even in debug mode" OFF)
+
 # options for security fuzzing
 # build configuration for fuzz testing is in onnxruntime_fuzz_test.cmake
 option(onnxruntime_FUZZ_TEST "Enable Fuzz testing" OFF)
diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
index 13298f984b..12dd6e0053 100644
--- a/cmake/onnxruntime_unittests.cmake
+++ b/cmake/onnxruntime_unittests.cmake
@@ -534,6 +534,9 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} )
   # the default logger tests conflict with the need to have an overall default logger
   # so skip in this type of
   target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS)
+  if(onnxruntime_RUN_MODELTEST_IN_DEBUG_MODE)
+    target_compile_definitions(onnxruntime_test_all PUBLIC -DRUN_MODELTEST_IN_DEBUG_MODE)
+  endif()
   if (onnxruntime_USE_FEATURIZERS)
     target_include_directories(onnxruntime_test_all PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/FeaturizersLibrary/src)
   endif()
@@ -889,4 +892,4 @@ if (onnxruntime_BUILD_JAVA)
     set_property(TEST onnxruntime4j_test APPEND PROPERTY DEPENDS onnxruntime4j_jni)
 endif()
 
-include(onnxruntime_fuzz_test.cmake)
\ No newline at end of file
+include(onnxruntime_fuzz_test.cmake)
diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc
index aa1a5233b5..9414b74759 100644
--- a/onnxruntime/test/providers/cpu/model_tests.cc
+++ b/onnxruntime/test/providers/cpu/model_tests.cc
@@ -772,7 +772,7 @@ TEST_P(ModelTest, Run) {
 #endif
 
     std::vector<std::basic_string<ORTCHAR_T>> paths;
-#ifdef NDEBUG
+#if defined(NDEBUG) || defined(RUN_MODELTEST_IN_DEBUG_MODE)
 #ifdef _WIN32
     paths.push_back(ORT_TSTR("..\\models"));
 #else
diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
index d190108f58..ffc368ebfe 100644
--- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
@@ -32,7 +32,7 @@ jobs:
 - job: Linux_C_API_Packaging_GPU_x64
   workspace:
     clean: all
-  timeoutInMinutes:  60
+  timeoutInMinutes:  120
   pool: 'Linux-GPU-CUDA10'
   steps:
     - template: templates/set-version-number-variables-step.yml
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml
index 7d883ae59d..5aef7b622e 100644
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml
@@ -25,6 +25,7 @@ jobs:
 - job: 'Linux_CI_Dev'
   workspace:
     clean: all
+  timeoutInMinutes: 120
   pool: $(AgentPoolLinux)
   steps:
     - template: ../../templates/set-version-number-variables-step.yml
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml
index 73fcd6064b..ab4444f7f3 100644
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml
@@ -67,6 +67,7 @@ jobs:
 - job: 'Linux_CI_Dev'
   workspace:
     clean: all
+  timeoutInMinutes: 120
   pool: $(AgentPoolLinux)
   steps:    
     - template: ../../templates/set-version-number-variables-step.yml
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
index b186dcc4b8..b8905bab9d 100644
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
@@ -90,6 +90,7 @@ jobs:
 - job: 'Linux_CI_Dev'
   workspace:
     clean: all
+  timeoutInMinutes: 120
   pool: $(AgentPoolLinux)
   steps:    
     - template: ../../templates/set-version-number-variables-step.yml
@@ -216,6 +217,10 @@ jobs:
       artifactName: 'drop-signed-nuget'
       targetPath: '$(Build.ArtifactStagingDirectory)'
 
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
+
 - template: test_all_os.yml
 
 - job: Publish_NuGet_Package_And_Report
@@ -305,3 +310,7 @@ jobs:
       command: 'push'
       packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.Managed.$(NuGetPackageVersionNumber).nupkg'
       publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7'
+
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
index c7606f68c9..0a53571f76 100644
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
@@ -16,6 +16,8 @@ jobs:
     value: '$(Build.BinariesDirectory)'
   - name: SKIPNONPACKAGETESTS
     value: 'ON'
+  - name: runCodesignValidationInjection
+    value: false
 
   steps:
   - task: UsePythonVersion@0
diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
index 6322db53be..ffcaf67b49 100644
--- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
+++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
@@ -3,6 +3,8 @@ jobs:
   workspace:
     clean: all
   timeoutInMinutes:  120
+  variables:
+    skipComponentGovernanceDetection: true
   pool: 'Linux-CPU'
   steps:
     - template: templates/set-version-number-variables-step.yml
@@ -15,12 +17,10 @@ jobs:
           sudo python3 -m pip install mysql-connector-python
           ln -s /data/models .
           #Build onnxruntime and run the instrumented program(unitests)
-          LLVM_PROFILE_FILE="%p.profraw" CFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CXXFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CC=clang CXX=clang++  python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir=$(Build.BinariesDirectory) --config Debug --parallel --skip_submodule_sync --build_shared_lib --enable_onnx_tests
+          LLVM_PROFILE_FILE="%p.profraw" CFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CXXFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CC=clang CXX=clang++  python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir=$(Build.BinariesDirectory) --config Debug --parallel --skip_submodule_sync --build_shared_lib --enable_onnx_tests --cmake_extra_defines RUN_MODELTEST_IN_DEBUG_MODE=ON
 
           cd Debug
           ./onnxruntime_mlas_test
-          #Run the model tests(It will take long because it is in debug mode)
-          LLVM_PROFILE_FILE="testrunner.profraw" ./onnx_test_runner -j 1 -c 2 /data/models
           #Merge the mulitple prof data into a single indexed profile data file
           llvm-profdata merge -sparse -o ort.profdata *.profraw
           #Create coverage report, output the result to 'report.json'
@@ -31,9 +31,11 @@ jobs:
       displayName: 'Post code coverage data to the Dashboard'
       inputs:
         script: |
-          python3 $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py  --commit_hash=$(OnnxRuntimeGitCommitHash) --report_file="$(Build.BinariesDirectory)/report.json" --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)"
+          python3 $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py  --commit_hash=$(OnnxRuntimeGitCommitHash) --report_file="$(Build.BinariesDirectory)/report.json" --report_url="https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=$(Build.BuildId)"
         workingDirectory: $(Build.BinariesDirectory)
       env: 
         DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password)
 
-    - template: templates/clean-agent-build-directory-step.yml
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml
index afce749d2a..9fa4134d12 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml
@@ -40,6 +40,7 @@ jobs:
     OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
     DotNetExe: 'dotnet.exe'
     CUDA_VERSION: ${{ parameters.CudaVersion }}
+    runCodesignValidationInjection: and(${{ parameters.DoNodejsPack }},${{ parameters. DoEsrp}}) #For the others, code sign is in a separated job
     DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     ${{ if eq(parameters.EnableLto, true) }}:
       build_py_lto_flag: --enable_lto
@@ -195,9 +196,10 @@ jobs:
       # Esrp signing
       - template: win-esrp-dll.yml
         parameters:
-          FolderPath: '$(Build.BinariesDirectory)\$(BuildConfig)'
+          FolderPath: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
           DisplayName: 'ESRP - Sign Native dlls'
           DoEsrp: ${{ parameters.DoEsrp }}
+          Pattern: 'onnx_test_runner.exe, onnxruntime_perf_test.exe,*.dll' #keep sync with src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
 
       - template: win-esrp-dll.yml
         parameters:
@@ -253,6 +255,13 @@ jobs:
           artifactName: ${{ parameters.ArtifactName }}
           targetPath: '$(Build.ArtifactStagingDirectory)'
 
+      # Put an unzipped version there to check if all the binaries are signed.
+      - script: |
+         7z x $(Build.ArtifactStagingDirectory)\prebuilds\onnxruntime-*.tar.gz
+         7z x $(Build.ArtifactStagingDirectory)\onnxruntime-*.tar
+        displayName: 'Unzip package to test'
+        workingDirectory: '$(Build.ArtifactStagingDirectory)'
+
     # Compliance tasks require logs from Debug Build
     - ${{ if eq(parameters['DoCompliance'], 'true') }}:
       - template: compliance.yml
@@ -261,4 +270,6 @@ jobs:
       parameters :
         condition : 'succeeded'
 
-    - template: clean-agent-build-directory-step.yml
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
index d5106a25f8..1166b99555 100644
--- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@@ -146,7 +146,9 @@ jobs:
     parameters :
       condition : 'succeeded'
 
-  - template: templates/clean-agent-build-directory-step.yml
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
 
 - job: 'x86_build'
   pool: 'Win-CPU-2019'
@@ -282,7 +284,9 @@ jobs:
     parameters :
       condition : 'succeeded'
 
-  - template: templates/clean-agent-build-directory-step.yml
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
    
 - job: 'x86_no_contrib_ops'
   pool: 'Win-CPU-2019'
@@ -407,7 +411,9 @@ jobs:
     parameters :
       condition : 'succeeded'
 
-  - template: templates/clean-agent-build-directory-step.yml
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
 
 - job: 'build_x64_no_contrib_ops'
   pool: 'Win-CPU-2019'
@@ -534,5 +540,7 @@ jobs:
     parameters :
       condition : 'succeeded'
 
-  - template: templates/clean-agent-build-directory-step.yml
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()