diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 26d6e3468a..ab579500c0 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1909,6 +1909,10 @@ def main(): if args.build: if args.build_wheel: nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') + wheel_name_suffix = args.wheel_name_suffix + if not args.use_openmp and wheel_name_suffix is None: + wheel_name_suffix = 'noopenmp' + build_python_wheel( source_dir, build_dir, @@ -1923,7 +1927,7 @@ def main(): args.use_acl, args.use_armnn, args.use_dml, - args.wheel_name_suffix, + wheel_name_suffix, args.enable_training, nightly_build=nightly_build, featurizers_build=args.use_featurizers, diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 46672b3790..7df7398669 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -141,6 +141,82 @@ stages: - template: clean-agent-build-directory-step.yml + - job: Linux_py_Wheels_no_openmp + timeoutInMinutes: 90 + workspace: + clean: all + pool: Linux-CPU + strategy: + matrix: + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + steps: + - checkout: self + clean: true + submodules: recursive + + - template: set-py-packaging-variables-step.yml + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch9j \ + $(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Release \ + --skip_submodule_sync \ + --parallel \ + --enable_lto \ + --build_wheel \ + --enable_onnx_tests \ + ${{ parameters.build_py_parameters }} \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)' + Contents: 'Release/dist/*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + ArtifactName: onnxruntime + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - template: clean-agent-build-directory-step.yml + - ${{ if eq(parameters.enable_linux_gpu, true) }}: - job: Linux_py_GPU_Wheels timeoutInMinutes: 120 @@ -424,6 +500,132 @@ stages: displayName: 'Clean Agent Directories' condition: always() + - job: Windows_py_Wheels_no_openmp + pool: 'Win-CPU-2019' + strategy: + matrix: + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + variables: + OrtPackageId: 'Microsoft.ML.OnnxRuntime' + MsbuildArguments: '-maxcpucount' + OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + EnvSetupScript: setup_env.bat + buildArch: x64 + setVcvars: true + BuildConfig: 'Release' + GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + timeoutInMinutes: 120 + workspace: + clean: all + + steps: + - checkout: self + clean: true + submodules: recursive + + - template: telemetry-steps.yml + + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + addToPath: true + architecture: 'x64' + + - template: set-nightly-build-option-variable-step.yml + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6 + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" + python setup.py bdist_wheel + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' + + - task: PythonScript@0 + displayName: 'BUILD' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: > + --config RelWithDebInfo + --enable_lto + --disable_rtti + --build_dir $(Build.BinariesDirectory) + --skip_submodule_sync + --cmake_generator "Visual Studio 16 2019" + --enable_pybind + --enable_onnx_tests + ${{ parameters.build_py_parameters }} + --parallel + $(TelemetryOption) + workingDirectory: '$(Build.BinariesDirectory)' + + # Esrp signing + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: true + Pattern: '*.pyd,*.dll' + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption) --wheel_name_suffix=noopenmp' + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - script: | + rename *.whl *.zip + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + displayName: 'Rename file extension for codesign validation' + + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**\*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + ArtifactName: onnxruntime + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() - ${{ if eq(parameters.enable_windows_gpu, true) }}: - job: Windows_py_GPU_Wheels workspace: diff --git a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh index 118f6d4eaa..91cb27218e 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh @@ -32,7 +32,7 @@ python3 /onnxruntime_src/tools/python/convert_onnx_models_to_ort.py \ find /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data -type f -name "*.onnx" -delete # Uninstall the ORT python wheel -python3 -m pip uninstall -y onnxruntime +python3 -m pip uninstall -y onnxruntime_noopenmp # Clear the build rm -rf /build/Debug