Fix component governance and codesign validation errors (#4277)

Adjust the job steps so that these security tasks run before the build directory clean up.

tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml

@@ -152,6 +152,7 @@ jobs:
     buildArch: x64
     setVcvars: true
     BuildConfig: 'Release'
+    GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
   timeoutInMinutes: 60
   workspace:
     clean: all
@@ -225,6 +226,20 @@ jobs:
       Contents: '**\dist\*.whl'
       TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
+  - script: |
+     rename *.whl *.zip
+
+    workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
+    displayName: 'Rename file extension for codesign validation'
+
+  - task: PublishTestResults@2
+    displayName: 'Publish unit test results'
+    inputs:
+      testResultsFiles: '**\*.results.xml'
+      searchFolder: '$(Build.BinariesDirectory)'
+      testRunTitle: 'Unit Test Run'
+    condition: succeededOrFailed()
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: ONNXRuntime python wheel'
     inputs:
@@ -234,7 +249,10 @@ jobs:
     parameters :
       condition : 'succeeded'
 
-  - template: templates/clean-agent-build-directory-step.yml
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
+
 
 - job:  Windows_py_GPU_Wheels
   workspace:
@@ -245,6 +263,7 @@ jobs:
     CUDA_VERSION: '10.1'
     buildArch: x64
     EnvSetupScript: setup_env_cuda.bat
+    GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
   strategy:
     matrix:
       Python35:
@@ -322,6 +341,20 @@ jobs:
         Contents: '**\dist\*.whl'
         TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
+    - script: |
+       rename *.whl *.zip
+
+      workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
+      displayName: 'Rename file extension for codesign validation'
+
+    - task: PublishTestResults@2
+      displayName: 'Publish unit test results'
+      inputs:
+        testResultsFiles: '**\*.results.xml'
+        searchFolder: '$(Build.BinariesDirectory)'
+        testRunTitle: 'Unit Test Run'
+      condition: succeededOrFailed()
+
     - task: PublishBuildArtifacts@1
       displayName: 'Publish Artifact: ONNXRuntime python wheel'
       inputs:
@@ -329,7 +362,9 @@ jobs:
 
     - template: templates/component-governance-component-detection-steps.yml
 
-    - template: templates/clean-agent-build-directory-step.yml
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
 
 - job: MacOS_py_Wheels
   workspace:

tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml

@@ -23,7 +23,12 @@ jobs:
           artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)'
           libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
           commitId: $(OnnxRuntimeGitCommitHash)
-    
+
+    - template: templates/component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
+    - template: templates/clean-agent-build-directory-step.yml
+
 - job: Linux_C_API_Packaging_GPU_x64
   workspace:
     clean: all
@@ -47,6 +52,9 @@ jobs:
           artifactName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion)'
           libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
           commitId: $(OnnxRuntimeGitCommitHash)
+    - template: templates/component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
     - template: templates/clean-agent-build-directory-step.yml
  
 
@@ -70,6 +78,9 @@ jobs:
           libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib'
           commitId: $(OnnxRuntimeGitCommitHash)
           
+    - template: templates/component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
     - template: templates/clean-agent-build-directory-step.yml
 
 - job: Windows_Packaging_CPU
@@ -124,6 +135,8 @@ jobs:
 
     - template: templates/set-test-data-variables-step.yml
     - template: templates/set-version-number-variables-step.yml
+
+
     - task: PythonScript@0
       displayName: 'Generate cmake config'
       inputs:
@@ -144,14 +157,6 @@ jobs:
         workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
         createLogFile: true
 
-    # Esrp signing
-    - template: templates/win-esrp-dll.yml
-      parameters:
-        FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
-        DisplayName: 'ESRP - Sign Native dlls'
-        DoEsrp: true
-        Pattern: '*.dll,*.exe'
-
     - task: PythonScript@0
       displayName: 'test'
       inputs:
@@ -164,8 +169,13 @@ jobs:
         buildConfig: RelWithDebInfo
         artifactName: 'onnxruntime-win-$(buildArch)-$(OnnxRuntimeVersion)'
         commitId: $(OnnxRuntimeGitCommitHash)
-    - template: templates/clean-agent-build-directory-step.yml
+    - template: templates/component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
 
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
 
 - job: Windows_Packaging_GPU
   workspace:
@@ -181,6 +191,7 @@ jobs:
     buildparameter: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda"
   steps:
     - template: templates/enable-telemetry.yml
+
     - task: UsePythonVersion@0
       inputs: 
         versionSpec: '3.7' 
@@ -230,14 +241,6 @@ jobs:
         workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
         createLogFile: true
 
-    # Esrp signing
-    - template: templates/win-esrp-dll.yml
-      parameters:
-        FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
-        DisplayName: 'ESRP - Sign Native dlls'
-        DoEsrp: true
-        Pattern: '*.dll,*.exe'
-
     - task: PythonScript@0
       displayName: 'test'
       inputs:
@@ -250,4 +253,9 @@ jobs:
         buildConfig: RelWithDebInfo
         artifactName: 'onnxruntime-win-$(buildArch)-gpu-$(OnnxRuntimeVersion)'
         commitId: $(OnnxRuntimeGitCommitHash)
-    - template: templates/clean-agent-build-directory-step.yml
+    - template: templates/component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()

tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml

@@ -21,17 +21,25 @@ steps:
           copy $(Build.SourcesDirectory)\include\onnxruntime\core\providers\cpu\cpu_provider_factory.h  $(Build.BinariesDirectory)\${{parameters.artifactName}}\include
           copy $(Build.SourcesDirectory)\include\onnxruntime\core\providers\cuda\cuda_provider_factory.h  $(Build.BinariesDirectory)\${{parameters.artifactName}}\include
 
-          # copy the README, licence and TPN
+          REM copy the README, licence and TPN
           copy $(Build.SourcesDirectory)\README.md $(Build.BinariesDirectory)\${{parameters.artifactName}}\README.md
           copy $(Build.SourcesDirectory)\docs\C_API.md $(Build.BinariesDirectory)\${{parameters.artifactName}}\C_API.md
           copy $(Build.SourcesDirectory)\docs\Privacy.md $(Build.BinariesDirectory)\${{parameters.artifactName}}\Privacy.md
           copy $(Build.SourcesDirectory)\LICENSE $(Build.BinariesDirectory)\${{parameters.artifactName}}\LICENSE
           copy $(Build.SourcesDirectory)\ThirdPartyNotices.txt $(Build.BinariesDirectory)\${{parameters.artifactName}}\ThirdPartyNotices.txt
-          copy $(Build.SourcesDirectory)/VERSION_NUMBER $(Build.BinariesDirectory)\${{parameters.artifactName}}\VERSION_NUMBER
+          copy $(Build.SourcesDirectory)\VERSION_NUMBER $(Build.BinariesDirectory)\${{parameters.artifactName}}\VERSION_NUMBER
           @echo ${{parameters.commitId}} > $(Build.BinariesDirectory)\${{parameters.artifactName}}\GIT_COMMIT_ID
 
         workingDirectory: '$(Build.BinariesDirectory)\${{parameters.buildConfig}}'
 
+
+    - template: win-esrp-dll.yml
+      parameters:
+        FolderPath: '$(Build.BinariesDirectory)\${{parameters.artifactName}}'
+        DisplayName: 'ESRP - Sign Native dlls'
+        DoEsrp: true
+        Pattern: '*.dll,*.exe'
+
     - task: ArchiveFiles@2
       inputs:
         rootFolderOrFile: '$(Build.BinariesDirectory)\${{parameters.artifactName}}' 
@@ -39,7 +47,7 @@ steps:
         archiveType: 'zip' # Options: zip, 7z, tar, wim
         archiveFile: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' 
         replaceExistingArchive: true 
-      
+
     - task: PublishBuildArtifacts@1
       inputs:
         pathtoPublish: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' 

tools/ci_build/github/azure-pipelines/templates/component-governance-component-detection-steps.yml

@@ -8,8 +8,8 @@ steps:
 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
   displayName: 'Component Detection'
   condition:
-    or(or(and(eq('${{parameters.condition}}', 'ci_only'), and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))),
+    or(or(and(eq('${{parameters.condition}}', 'ci_only'), and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Scheduled'))),
           and(eq('${{parameters.condition}}', 'always'), always())),
           and(eq('${{parameters.condition}}', 'succeeded'), succeeded()))
   inputs:
-    ignoreDirectories: '$(Build.SourcesDirectory)/cmake/external/tvm/3rdparty/dmlc-core/tracker' # ignore dmlc-core tracker for its CI, which is not used in onnxruntime build
+    ignoreDirectories: '$(Build.SourcesDirectory)/cmake/external/tvm/3rdparty/dmlc-core/tracker' # ignore dmlc-core tracker for its CI, which is not used in onnxruntime build