From e7a0495a874251e9747b2ce0683e0580282c54df Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 18 Sep 2023 15:03:48 -0700 Subject: [PATCH] Cherry-picks pipeline changes to 1.16.0 release branch (#17577) ### Description 1. Delete Prefast tasks (#17522) 2. Disable yum update (#17551) 3. Avoid calling patchelf (#17365 and #17562) we that we can validate the above fix The main problem I'm trying to solve is: our GPU package depends on both CUDA 11.x and CUDA 12.x . However, it's not easy to see the information because ldd doesn't work with the shared libraries we generate(see issue #9754) . So the patchelf change are useful for me to validate the "Disabling yum update" was successful. As you can see we call "yum update" from multiple places. Without some kind of validation it's hard to say if I have covered all of them. The Prefast change is needed because I'm going to update the VM images in the next a few weeks. In case of we need to publish a patch release after that. ### Motivation and Context Without this fix we will mix using CUDA 11.x and CUDA 12.x. And it will crash every time when we use TensorRT. --- .github/workflows/sca.yml | 133 ------------------ setup.py | 130 ++++------------- .../azure-pipelines/post-merge-jobs.yml | 5 - .../azure-pipelines/templates/compliance.yml | 21 --- .../templates/jobs/win-ci-vs-2022-job.yml | 48 ------- .../templates/py-packaging-stage.yml | 18 --- .../azure-pipelines/templates/py-win-gpu.yml | 59 -------- .../azure-pipelines/templates/win-ci.yml | 19 --- .../azure-pipelines/win-ci-pipeline.yml | 10 -- .../azure-pipelines/win-gpu-ci-pipeline.yml | 4 - .../linux/docker/Dockerfile.manylinux2014_cpu | 4 - .../docker/Dockerfile.manylinux2014_cuda11 | 3 - ...kerfile.manylinux2014_cuda11_6_tensorrt8_4 | 3 - ...kerfile.manylinux2014_cuda11_6_tensorrt8_5 | 3 - ...kerfile.manylinux2014_cuda11_8_tensorrt8_6 | 3 - .../docker/Dockerfile.manylinux2014_rocm | 3 - ...Dockerfile.manylinux2014_training_cuda11_8 | 3 - .../python/cpu/Dockerfile.manylinux2014_cpu | 3 - .../github/linux/docker/manylinux.patch | 25 +++- .../docker/scripts/manylinux/install_deps.sh | 2 +- 20 files changed, 56 insertions(+), 443 deletions(-) delete mode 100644 .github/workflows/sca.yml diff --git a/.github/workflows/sca.yml b/.github/workflows/sca.yml deleted file mode 100644 index 1416f5a4d3..0000000000 --- a/.github/workflows/sca.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: Windows_SCA -on: - push: - branches: - - main - - rel-* - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 - -jobs: - Onnxruntime-SCA-training-CUDA: - runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"] - steps: - - uses: actions/checkout@v3 - with: - submodules: false - - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - architecture: 'x64' - - - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Download cuda - run: azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" cuda_sdk - - - - name: Delete build folder - run: | - if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b } - &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug - - # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter. - - name: Build code - env: - CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake' - run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v11.8 --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 - - - name: Generate sarif - working-directory: D:\b - run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output - - - name: Upload SARIF to GitHub - uses: github/codeql-action/upload-sarif@v2 - continue-on-error: true - with: - sarif_file: ${{ github.workspace }}\output\MergeResult.sarif - category: VS_SCA - - # No python - Onnxruntime-SCA-win32-WINML-x64: - runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"] - steps: - - uses: actions/checkout@v3 - with: - submodules: false - - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - architecture: 'x64' - - - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Delete build folder - run: | - if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b } - &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug - - # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter. - - name: Build code - env: - CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake' - run: python tools\ci_build\build.py --build_java --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib - - - name: Generate sarif - working-directory: D:\b - run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output - - - name: Upload SARIF to GitHub - uses: github/codeql-action/upload-sarif@v2 - continue-on-error: true - with: - sarif_file: ${{ github.workspace }}\output\MergeResult.sarif - category: VS_SCA_WIN32_WINML_X64 - - # No java, No python - Onnxruntime-SCA-win32-WINML-x86: - runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"] - steps: - - uses: actions/checkout@v3 - with: - submodules: false - - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - architecture: 'x86' - - - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Delete build folder - run: | - if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b } - &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x86 -install_prefix D:\b\Debug\installed -build_config Debug - - # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter. - - name: Build code - env: - CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake' - run: python tools\ci_build\build.py --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib - - - name: Generate sarif - working-directory: D:\b - run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output - - - name: Upload SARIF to GitHub - uses: github/codeql-action/upload-sarif@v2 - continue-on-error: true - with: - sarif_file: ${{ github.workspace }}\output\MergeResult.sarif - category: VS_SCA_WIN32_WINML_X86 diff --git a/setup.py b/setup.py index c4bbb67947..3a16c38aec 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import datetime import logging import platform +import shlex import subprocess import sys from glob import glob, iglob @@ -183,108 +184,37 @@ try: dest = "onnxruntime/capi/onnxruntime_pybind11_state_manylinux1.so" logger.info("copying %s -> %s", source, dest) copyfile(source, dest) - result = subprocess.run( - ["patchelf", "--print-needed", dest], check=True, stdout=subprocess.PIPE, text=True - ) - dependencies = [ - "librccl.so", - "libamdhip64.so", - "librocblas.so", - "libMIOpen.so", - "libhsa-runtime64.so", - "libhsakmt.so", - ] + to_preload = [] to_preload_cuda = [] to_preload_tensorrt = [] to_preload_cann = [] - cuda_dependencies = [] - args = ["patchelf", "--debug"] - for line in result.stdout.split("\n"): - for dependency in dependencies: - if dependency in line: - to_preload.append(line) - args.extend(["--remove-needed", line]) - args.append(dest) - if len(args) > 3: - subprocess.run(args, check=True, stdout=subprocess.PIPE) - dest = "onnxruntime/capi/libonnxruntime_providers_" + ("rocm.so" if is_rocm else "cuda.so") - if path.isfile(dest): - result = subprocess.run( - ["patchelf", "--print-needed", dest], - check=True, - stdout=subprocess.PIPE, - text=True, - ) - cuda_dependencies = [ - "libcublas.so", - "libcublasLt.so", - "libcudnn.so", - "libcudart.so", - "libcurand.so", - "libcufft.so", - "libnvToolsExt.so", - "libcupti.so", - ] - rocm_dependencies = [ - "librccl.so", - "libamdhip64.so", - "librocblas.so", - "libMIOpen.so", - "libhsa-runtime64.so", - "libhsakmt.so", - ] - args = ["patchelf", "--debug"] - for line in result.stdout.split("\n"): - for dependency in cuda_dependencies + rocm_dependencies: - if dependency in line: - if dependency not in to_preload: - to_preload_cuda.append(line) - args.extend(["--remove-needed", line]) - args.append(dest) - if len(args) > 3: - subprocess.run(args, check=True, stdout=subprocess.PIPE) + cuda_dependencies = [ + "libcublas.so.11", + "libcublasLt.so.11", + "libcudnn.so.8", + "libcudart.so.11.0", + "libcurand.so.10", + "libcufft.so.10", + ] + rocm_dependencies = [ + "librccl.so.1", + "libnuma.so.1", + "libamd_comgr.so.2", + "libdrm.so.2", + "librocblas.so.0", + "libdrm_amdgpu.so.1", + "libamdhip64.so.5", + "libroctracer64.so.4", + "libMIOpen.so.1", + "libtinfo.so.6", + "libelf.so.1", + "librocm_smi64.so.5", + "libhsa-runtime64.so.1", + ] - dest = "onnxruntime/capi/libonnxruntime_providers_" + ("migraphx.so" if is_rocm else "tensorrt.so") - if path.isfile(dest): - result = subprocess.run( - ["patchelf", "--print-needed", dest], - check=True, - stdout=subprocess.PIPE, - text=True, - ) - tensorrt_dependencies = ["libnvinfer.so", "libnvinfer_plugin.so", "libnvonnxparser.so"] - args = ["patchelf", "--debug"] - for line in result.stdout.split("\n"): - for dependency in cuda_dependencies + tensorrt_dependencies: - if dependency in line: - if dependency not in (to_preload + to_preload_cuda): - to_preload_tensorrt.append(line) - args.extend(["--remove-needed", line]) - args.append(dest) - if len(args) > 3: - subprocess.run(args, check=True, stdout=subprocess.PIPE) - - dest = "onnxruntime/capi/libonnxruntime_providers_cann.so" - if path.isfile(dest): - result = subprocess.run( - ["patchelf", "--print-needed", dest], - check=True, - stdout=subprocess.PIPE, - text=True, - ) - cann_dependencies = ["libascendcl.so", "libacl_op_compiler.so", "libfmk_onnx_parser.so"] - args = ["patchelf", "--debug"] - for line in result.stdout.split("\n"): - for dependency in cann_dependencies: - if dependency in line: - if dependency not in to_preload: - to_preload_cann.append(line) - args.extend(["--remove-needed", line]) - args.append(dest) - if len(args) > 3: - subprocess.run(args, check=True, stdout=subprocess.PIPE) + tensorrt_dependencies = ["libnvinfer.so.8", "libnvinfer_plugin.so.8", "libnvonnxparser.so.8"] dest = "onnxruntime/capi/libonnxruntime_providers_openvino.so" if path.isfile(dest): @@ -308,10 +238,12 @@ try: assert self.dist_dir is not None file = glob(path.join(self.dist_dir, "*linux*.whl"))[0] logger.info("repairing %s for manylinux1", file) + auditwheel_cmd = ["auditwheel", "-v", "repair", "-w", self.dist_dir, file] + for i in cuda_dependencies + rocm_dependencies + tensorrt_dependencies: + auditwheel_cmd += ["--exclude", i] + logger.info("Running {}".format(" ".join([shlex.quote(arg) for arg in auditwheel_cmd]))) try: - subprocess.run( - ["auditwheel", "repair", "-w", self.dist_dir, file], check=True, stdout=subprocess.PIPE - ) + subprocess.run(auditwheel_cmd, check=True, stdout=subprocess.PIPE) finally: logger.info("removing %s", file) remove(file) diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index 872812a6ae..b858770583 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -74,7 +74,6 @@ stages: isX86: false job_name_suffix: x64_RelWithDebInfo RunOnnxRuntimeTests: true - RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA WITH_CACHE: true MachinePool: onnxruntime-Win2022-GPU-MultiA10 @@ -93,7 +92,6 @@ stages: isX86: false job_name_suffix: x64_mimalloc RunOnnxRuntimeTests: true - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false @@ -113,7 +111,6 @@ stages: isX86: false job_name_suffix: x64_no_memory_profiling RunOnnxRuntimeTests: false - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false @@ -133,7 +130,6 @@ stages: isX86: false job_name_suffix: x64_minimal_no_exception RunOnnxRuntimeTests: true - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false @@ -153,7 +149,6 @@ stages: isX86: false job_name_suffix: x64_debug_node_input_output RunOnnxRuntimeTests: true - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml index 04d999b556..0dfe398c8b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/compliance.yml +++ b/tools/ci_build/github/azure-pipelines/templates/compliance.yml @@ -18,27 +18,6 @@ steps: arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.dll --recurse --verbose' continueOnError: true -- task: DeleteFiles@1 - displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - Contents: | - **/*.obj - **/*.pdb - **/*.dll - -# Manually set msBuildCommandline so that we can also set CAExcludePath -- task: SDLNativeRules@3 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - inputs: - userProvideBuildInfo: msBuildInfo - msBuildArchitecture: x64 - msBuildVersion: 17.0 - msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files' - rulesetName: Custom - customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset - - task: SdtReport@2 displayName: 'Create Security Analysis Report' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml index 1cd21ea199..82bbebf133 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml @@ -34,11 +34,6 @@ parameters: type: boolean default: true -- name: RunStaticCodeAnalysis - displayName: Run Static Code Analysis - type: boolean - default: true - - name: ORT_EP_NAME type: string @@ -309,49 +304,6 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' displayName: 'Run tests' - - - ${{ if eq(parameters.RunStaticCodeAnalysis, true) }}: - - task: DeleteFiles@1 - displayName: 'Delete binaries files from $(Build.BinariesDirectory)\RelWithDebInfo' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - Contents: | - **/*.obj - **/*.pdb - **/*.dll - - - # Manually set msBuildCommandline so that we can also set CAExcludePath - # build_dir must be a sub folder of $(Build.SourcesDirectory) - # TODO: move this step to a CPU-only machine to save GPU resources. - - task: SDLNativeRules@3 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - inputs: - msBuildArchitecture: amd64 - setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON onnxruntime_ENABLE_LTO=OFF' - msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.SourcesDirectory)\b\RelWithDebInfo\onnxruntime.sln" /p:RunCodeAnalysis=true /p:platform=${{ parameters.msbuildPlatform }} /p:configuration=RelWithDebInfo /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files' - rulesetName: Custom - customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset - publishXML: true - - - task: SdtReport@2 - displayName: 'Create Security Analysis Report' - inputs: - SDLNativeRules: true - - - task: PublishSecurityAnalysisLogs@3 - displayName: 'Publish Security Analysis Logs' - continueOnError: true - - - task: PostAnalysis@2 - displayName: 'Guardian Break v2' - inputs: - GdnBreakGdnToolSDLNativeRulesSeverity: Note - GdnBreakGdnToolSDLNativeRules: true - - - - ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}: - task: PublishTestResults@2 displayName: 'Publish unit test results' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 568ab6c8a8..a872228f5d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -246,24 +246,6 @@ stages: workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run Python Tests' - #Skip it for 32 bits x86 build. Currently the scan tool has a bug: it doesn't allow me use 64 bits link.exe - #in 32 bits Win32 build. I tried all the settings but they all don't work. - - task: SDLNativeRules@3 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - condition: and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8'))) - inputs: - msBuildArchitecture: amd64 - setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --enable_pybind --enable_onnx_tests --parallel $(TelemetryOption) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON onnxruntime_ENABLE_LTO=OFF' - msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.SourcesDirectory)\b\Debug\onnxruntime.sln" /p:RunCodeAnalysis=true /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files' - rulesetName: Custom - customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset - - - task: SdtReport@2 - displayName: 'Create Security Analysis Report' - inputs: - SDLNativeRules: true - - task: TSAUpload@2 displayName: 'TSA upload' condition: and(and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8'))), eq(variables['Build.SourceBranch'], 'refs/heads/main')) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index ef938a6345..919749cac1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -22,65 +22,6 @@ parameters: default: '' jobs: -- ${{ if eq(parameters.PYTHON_VERSION, '3.8') }}: - - job: Win_py_${{ parameters.EP_NAME }}_Wheels_StaticAnalysis - timeoutInMinutes: 240 - workspace: - clean: all - pool: onnxruntime-Win-CPU-2022 - steps: - - checkout: self - clean: true - submodules: none - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.8 - addToPath: true - architecture: 'x64' - - task: onebranch.pipeline.tsaoptions@1 - displayName: 'OneBranch TSAOptions' - inputs: - tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' - appendSourceBranchName: false - - - template: download-deps.yml - - - template: jobs/set-winenv.yml - parameters: - EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }} - DownloadCUDA: true - - - task: PythonScript@0 - displayName: 'Update deps.txt' - inputs: - scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py - arguments: --new_dir $(Build.BinariesDirectory)/deps - workingDirectory: $(Build.BinariesDirectory) - - - task: SDLNativeRules@3 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - inputs: - msBuildArchitecture: amd64 - setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --enable_pybind ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON onnxruntime_ENABLE_LTO=OFF' - msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.SourcesDirectory)\b\Debug\onnxruntime.sln" /p:RunCodeAnalysis=true /p:platform=x64 /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files' - rulesetName: Custom - customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset - publishXML: true - - - task: SdtReport@2 - displayName: 'Create Security Analysis Report' - inputs: - SDLNativeRules: true - - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - - job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }} timeoutInMinutes: 240 workspace: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index f6da7bb857..80d285f3fd 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -263,25 +263,6 @@ stages: AnalyzeTargetGlob: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\**\*.dll' continueOnError: true - - task: DeleteFiles@1 - displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - Contents: | - **/*.obj - **/*.pdb - **/*.dll - - #Manually set msBuildCommandline so that we can also set CAExcludePath - - task: SDLNativeRules@3 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - condition: and (succeeded(), eq(variables['msbuildPlatform'], 'x64')) - inputs: - msBuildArchitecture: amd64 - setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' - msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\Debug\onnxruntime.sln" /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' - - task: PostAnalysis@2 inputs: GdnBreakAllTools: false diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 7f71f41484..b7e3ce7940 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -44,7 +44,6 @@ stages: isX86: false job_name_suffix: x64_debug RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false @@ -66,7 +65,6 @@ stages: isX86: false job_name_suffix: x64_release RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false @@ -86,7 +84,6 @@ stages: isX86: false job_name_suffix: x64_release RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: DNNL GenerateDocumentation: false @@ -108,7 +105,6 @@ stages: isX86: false job_name_suffix: x64_release RunOnnxRuntimeTests: true - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: XNNPACK GenerateDocumentation: false @@ -129,7 +125,6 @@ stages: job_name_suffix: x64_release_winml RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} # WinML has many warnings - RunStaticCodeAnalysis: false EnablePython: false isTraining: false ORT_EP_NAME: CPU @@ -150,7 +145,6 @@ stages: isX86: true job_name_suffix: x86_release RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false @@ -170,7 +164,6 @@ stages: isX86: false job_name_suffix: training_x64_debug RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false isTraining: true ORT_EP_NAME: CPU GenerateDocumentation: false @@ -190,7 +183,6 @@ stages: isX86: false job_name_suffix: training_x64_release RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: true isTraining: true ORT_EP_NAME: CPU GenerateDocumentation: false @@ -210,7 +202,6 @@ stages: isX86: false job_name_suffix: ort_training_apis_x64_release RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false EnablePython: false isTraining: true ORT_EP_NAME: CPU @@ -231,7 +222,6 @@ stages: isX86: false job_name_suffix: x64_release_azure RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false EnablePython: false isTraining: false ORT_EP_NAME: CPU diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 7ab55a5d80..806ed797f8 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -49,7 +49,6 @@ stages: isX86: false job_name_suffix: x64_RelWithDebInfo RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA WITH_CACHE: true MachinePool: onnxruntime-Win2022-GPU-A10 @@ -67,7 +66,6 @@ stages: isX86: false job_name_suffix: x64_RelWithDebInfo RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA WITH_CACHE: true # Some unit tests crash on A10 GPUs. So this job still needs to use A10. @@ -87,7 +85,6 @@ stages: isX86: false job_name_suffix: x64_RelWithDebInfo RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - RunStaticCodeAnalysis: false ORT_EP_NAME: DML WITH_CACHE: true MachinePool: onnxruntime-Win2022-GPU-dml-A10 @@ -106,7 +103,6 @@ stages: isX86: false job_name_suffix: x64_RelWithDebInfo RunOnnxRuntimeTests: false - RunStaticCodeAnalysis: false GenerateDocumentation: true ORT_EP_NAME: CUDA # It doesn't really matter which EP is selected here since this stage is for documentation. WITH_CACHE: true diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu index 033afde6aa..561df220af 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu @@ -26,7 +26,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -35,7 +34,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -137,9 +135,7 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ - build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ build_scripts/requirements3.10.txt \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 index dc52fb51d6..8a092c437a 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 @@ -31,7 +31,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -40,7 +39,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -140,7 +138,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 index 303e83eb23..68b779e6f1 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 @@ -31,7 +31,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -40,7 +39,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -140,7 +138,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 index d17e4b2458..dfc9e819ad 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 @@ -31,7 +31,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -40,7 +39,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -140,7 +138,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_8_tensorrt8_6 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_8_tensorrt8_6 index bcdc24d5eb..6e27db4eb3 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_8_tensorrt8_6 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_8_tensorrt8_6 @@ -31,7 +31,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -40,7 +39,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -147,7 +145,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.7.txt \ build_scripts/requirements3.8.txt \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm index 9f7575d62e..036d261044 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm @@ -52,7 +52,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -61,7 +60,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -164,7 +162,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_8 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_8 index 5d77446007..c3c7213212 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_8 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_8 @@ -31,7 +31,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -40,7 +39,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -140,7 +138,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/Dockerfile.manylinux2014_cpu b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/Dockerfile.manylinux2014_cpu index 8869a78902..691e45e743 100644 --- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/Dockerfile.manylinux2014_cpu +++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/Dockerfile.manylinux2014_cpu @@ -26,7 +26,6 @@ COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors # setup entrypoint, this will wrap commands with `linux32` with i686 images COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ @@ -35,7 +34,6 @@ COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ build_scripts/build_utils.sh \ /build_scripts/ RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ @@ -132,7 +130,6 @@ COPY --from=build_git /manylinux-rootfs / COPY --from=build_cpython /manylinux-rootfs / COPY --from=all_python /opt/_internal /opt/_internal/ COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ build_scripts/python-tag-abi-tag.py \ build_scripts/requirements3.8.txt \ build_scripts/requirements3.9.txt \ diff --git a/tools/ci_build/github/linux/docker/manylinux.patch b/tools/ci_build/github/linux/docker/manylinux.patch index 7750118d01..1a92b4c094 100644 --- a/tools/ci_build/github/linux/docker/manylinux.patch +++ b/tools/ci_build/github/linux/docker/manylinux.patch @@ -50,6 +50,17 @@ index 961e34d..55ae11b 100755 make install > /dev/null } +diff --git a/finalize.sh b/finalize.sh +index 621eab9..4cbcf90 100755 +--- a/finalize.sh ++++ b/finalize.sh +@@ -86,6 +86,3 @@ clean_pyc /opt/_internal + rm -rf /root/.cache + + hardlink -cv /opt/_internal +- +-# update system packages +-LC_ALL=C ${MY_DIR}/update-system-packages.sh diff --git a/install-entrypoint.sh b/install-entrypoint.sh index 9ef1e99..ec52833 100755 --- a/install-entrypoint.sh @@ -65,7 +76,7 @@ index 9ef1e99..ec52833 100755 +fi \ No newline at end of file diff --git a/install-runtime-packages.sh b/install-runtime-packages.sh -index 137d2e2..21b60a7 100755 +index 137d2e2..7a17e16 100755 --- a/install-runtime-packages.sh +++ b/install-runtime-packages.sh @@ -73,9 +73,11 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then @@ -83,3 +94,15 @@ index 137d2e2..21b60a7 100755 elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then # Software collection (for devtoolset-10) yum -y install centos-release-scl-rh +@@ -121,11 +123,6 @@ else + exit 1 + fi + +-# update system packages, we already updated them but +-# the following script takes care of cleaning-up some things +-# and since it's also needed in the finalize step, everything's +-# centralized in this script to avoid code duplication +-LC_ALL=C ${MY_DIR}/update-system-packages.sh + + if [ "${BASE_POLICY}" == "manylinux" ]; then + # we'll be removing libcrypt.so.1 later on diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh index c34abbd2ba..d0aea8b448 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh @@ -3,7 +3,7 @@ set -e -x # Development tools and libraries if [ -f /etc/redhat-release ]; then - yum update && yum -y install graphviz + yum -y install graphviz os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) elif [ -f /etc/os-release ]; then apt-get update && apt-get install -y graphviz