From fdbbd20f3289b2878f2fbad3f77bff1ddd375b28 Mon Sep 17 00:00:00 2001 From: Remi Domingues Date: Fri, 30 Dec 2022 17:52:20 +0000 Subject: [PATCH] Cache conda and pip for IOS CI (#91359) Fixes T137630520 Caching for conda and pip dependencies for iOS CI workflow. - Conda and pip dependencies have been moved from [_ios-build-test.yml](https://github.com/pytorch/pytorch/blob/master/.github/workflows/_ios-build-test.yml) to dedicated requirements files - Miniconda shell installation has been replaced by `setup-miniconda@main` which supports caching Pull Request resolved: https://github.com/pytorch/pytorch/pull/91359 Approved by: https://github.com/malfet, https://github.com/huydhn --- .github/requirements/README.md | 4 ++ .github/requirements/conda-env-iOS | 11 +++++ .github/requirements/pip-requirements-iOS.txt | 4 ++ .github/workflows/_ios-build-test.yml | 40 +++++-------------- scripts/build_ios.sh | 17 +++++++- 5 files changed, 44 insertions(+), 32 deletions(-) create mode 100644 .github/requirements/conda-env-iOS create mode 100644 .github/requirements/pip-requirements-iOS.txt diff --git a/.github/requirements/README.md b/.github/requirements/README.md index 7300eee1456..28230a44d67 100644 --- a/.github/requirements/README.md +++ b/.github/requirements/README.md @@ -13,6 +13,8 @@ reliability. The list of support files are as follows: * Conda: + * conda-env-iOS. This is used by iOS build and test jobs to setup the + conda environment * conda-env-macOS-ARM64. This is used by MacOS (m1, arm64) build and test jobs to setup the conda environment * conda-env-macOS-X64. This is use by MacOS (x86-64) build and test @@ -20,5 +22,7 @@ The list of support files are as follows: * conda-env-Linux-X64. This is used by Linux buck build and test jobs to setup the conda environment * Pip: + * pip-requirements-iOS.txt. This is used by iOS build and test jobs to + setup the pip environment * pip-requirements-macOS.txt. This is used by MacOS build and test jobs to setup the pip environment diff --git a/.github/requirements/conda-env-iOS b/.github/requirements/conda-env-iOS new file mode 100644 index 00000000000..283f88cec49 --- /dev/null +++ b/.github/requirements/conda-env-iOS @@ -0,0 +1,11 @@ +blas=1.0 +cffi=1.15.1 +cmake=3.22.1 +mkl=2022.1.0 +mkl-include=2022.1.0 +ninja=1.10.2 +numpy=1.23.3 +pyyaml=6.0 +requests=2.28.1 +setuptools=63.4.1 +typing_extensions=4.3.0 diff --git a/.github/requirements/pip-requirements-iOS.txt b/.github/requirements/pip-requirements-iOS.txt new file mode 100644 index 00000000000..cf720eaae33 --- /dev/null +++ b/.github/requirements/pip-requirements-iOS.txt @@ -0,0 +1,4 @@ +# iOS simulator requirements +coremltools==5.0b5 +protobuf==3.20.1 +six==1.16.0 diff --git a/.github/workflows/_ios-build-test.yml b/.github/workflows/_ios-build-test.yml index 2754a682d92..a90479983b7 100644 --- a/.github/workflows/_ios-build-test.yml +++ b/.github/workflows/_ios-build-test.yml @@ -67,27 +67,12 @@ jobs: # Install dependencies brew install libtool - - name: Install conda and dependencies - run: | - # Install conda, setup-miniconda messes with the path that messes with the ruby stuff we do later on - curl --retry 3 --retry-all-errors -o "${RUNNER_TEMP}/conda.sh" https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-MacOSX-x86_64.sh - chmod +x "${RUNNER_TEMP}/conda.sh" - /bin/bash "${RUNNER_TEMP}/conda.sh" -b -p "${RUNNER_TEMP}/anaconda" - echo "${RUNNER_TEMP}/anaconda/bin" >> "${GITHUB_PATH}" - # shellcheck disable=SC1091 - source "${RUNNER_TEMP}/anaconda/bin/activate" - conda install -y \ - blas=1.0 \ - cffi=1.15.1 \ - cmake=3.22.1 \ - mkl=2022.1.0 \ - mkl-include=2022.1.0 \ - ninja=1.10.2 \ - numpy=1.23.3 \ - pyyaml=6.0 \ - requests=2.28.1 \ - setuptools=63.4.1 \ - typing_extensions=4.3.0 + - name: Setup miniconda for iOS + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: "3.9" + environment-file: .github/requirements/conda-env-iOS + pip-requirements-file: .github/requirements/pip-requirements-iOS.txt - name: Setup Fastlane run: | @@ -100,11 +85,9 @@ jobs: - name: Build PyTorch Mobile Runtime run: | # shellcheck disable=SC1091 - source "${RUNNER_TEMP}/anaconda/bin/activate" export TCLLIBPATH="/usr/local/lib" python -VV - export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"} - scripts/build_ios.sh + ${CONDA_RUN} scripts/build_ios.sh - name: Build TestApp if: inputs.ios-platform == 'SIMULATOR' @@ -121,19 +104,16 @@ jobs: if: inputs.ios-platform == 'SIMULATOR' run: | # shellcheck disable=SC1091 - source "${RUNNER_TEMP}/anaconda/bin/activate" # use the pytorch nightly build to generate models - pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + ${CONDA_RUN} pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html # generate models for differnet backends cd "${GITHUB_WORKSPACE}/ios/TestApp/benchmark" mkdir -p ../models if [ "${USE_COREML_DELEGATE}" == 1 ]; then - pip install coremltools==5.0b5 protobuf==3.20.1 - pip install six==1.16.0 - python coreml_backend.py + ${CONDA_RUN} python coreml_backend.py else cd "${GITHUB_WORKSPACE}" - python test/mobile/model_test/gen_test_model.py ios-test + ${CONDA_RUN} python test/mobile/model_test/gen_test_model.py ios-test fi cd "${GITHUB_WORKSPACE}/ios/TestApp/benchmark" if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh index 335d14b5217..8262eeced09 100755 --- a/scripts/build_ios.sh +++ b/scripts/build_ios.sh @@ -9,11 +9,24 @@ CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)" +if [ -z "$PYTHON" ]; then + PYTHON=python + PYTHON_VERSION_MAJOR=$($PYTHON -c 'import sys; print(sys.version_info[0])') + if [ "${PYTHON_VERSION_MAJOR}" -le 2 ]; then + echo "Default python executable is Python-2, trying to use python3 alias" + PYTHON=python3 + fi +fi + +echo "Bash: $(/bin/bash --version | head -1)" +echo "Python: $($PYTHON -c 'import sys; print(sys.version)')" +echo "Caffe2 path: $CAFFE2_ROOT" + CMAKE_ARGS=() # Build PyTorch mobile -CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')") -CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')") +CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')") +CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')") CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF") # custom build with selected ops