From 04ffdc997d6dea9a2f66e29a6045d76e487804bd Mon Sep 17 00:00:00 2001 From: Cuong Duong Date: Sat, 25 Dec 2021 23:20:24 +1100 Subject: [PATCH] Add Windows wheel (#2089) --- .github/workflows/wheel.yml | 90 ++++++++++++++++++++++++++++++++++++- python/prophet/models.py | 17 +++++++ python/setup.py | 48 +++++++++++++------- 3 files changed, 139 insertions(+), 16 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index a241486..5e685bb 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -10,6 +10,94 @@ env: CMDSTAN_VERSION: "2.26.1" jobs: + make-wheel-windows: + name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["windows-latest"] + python-version: [3.8] + architecture: ["x64"] + fail-fast: false + + steps: + - name: "Setup environment variables (Windows)" + if: startsWith(runner.os, 'Windows') + shell: pwsh + run: | + (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled + $os_version = (Get-CimInstance Win32_OperatingSystem).version + Echo "OS_VERSION=$os_version" >> $env:GITHUB_ENV + Echo "PIP_DEFAULT_CACHE=$HOME/pip/cache" >> $env:GITHUB_ENV + Echo "DEFAULT_HOME=$HOME" >> $env:GITHUB_ENV + + - name: "Checkout repo" + uses: actions/checkout@v2 + + - name: "Set up Python" + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} + + - name: "Restore pip cache" + id: cache-pip + uses: actions/cache@v2 + with: + path: ${{ env.PIP_DEFAULT_CACHE }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/python/requirements.txt') }}-v1 + restore-keys: | + ${{ runner.os }}-pip- + + - name: "Install pip" + shell: pwsh + run: | + python -m pip install --upgrade pip + python -m pip install cibuildwheel build delvewheel + + - name: "Restore cmdstan cache" + id: cache-cmdstan + uses: actions/cache@v2 + with: + path: ${{ env.DEFAULT_HOME }}/.cmdstan + key: ${{ runner.os }}-cmdstan-${{ env.CMDSTAN_VERSION }}-v1 + + - name: "Restore RTools40" + id: cache-rtools + uses: actions/cache@v2 + with: + path: C:/rtools40 + key: ${{ runner.os }}-${{ env.OS_VERSION }}-rtools-v1 + + - name: "Download cmdstan" + if: steps.cache-cmdstan.outputs.cache-hit != 'true' + run: | + $ProgressPreference = "SilentlyContinue" + Invoke-WebRequest https://github.com/stan-dev/cmdstan/releases/download/v${{ env.CMDSTAN_VERSION }}/cmdstan-${{ env.CMDSTAN_VERSION }}.tar.gz -OutFile "D:/a/_temp/cmdstan.tar.gz" + New-Item -Path "$HOME" -Name ".cmdstan" -ItemType "directory" + tar -xf "D:/a/_temp/cmdstan.tar.gz" -C "$HOME/.cmdstan" + + - name: "Build wheel" + run: | + cd python && python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ENVIRONMENT: > + STAN_BACKEND="${{ env.STAN_BACKEND }}" + CMDSTAN_VERSION=${{ env.CMDSTAN_VERSION }} + PIP_CACHE_DIR="${{ env.PIP_DEFAULT_CACHE }}" + CIBW_BUILD: cp36-* cp37-* cp38-* + CIBW_ARCHS: native + CIBW_BUILD_FRONTEND: build + # CIBW_REPAIR_WHEEL_COMMAND: delvewheel repair -w {dest_dir} {wheel} + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest --pyargs prophet + + - name: "Upload wheel as artifact" + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.os }}-wheel + path: "./**/*.whl" + make-wheels-macos-linux: name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -18,7 +106,6 @@ jobs: os: ["macos-latest", "ubuntu-latest"] python-version: [3.8] architecture: ["x64"] - fail-fast: false steps: @@ -89,6 +176,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_BEFORE_ALL_LINUX: chmod -R a+rwx /host/${{ env.PIP_DEFAULT_CACHE }} CIBW_BUILD: cp36-* cp37-* cp38-* + CIBW_SKIP: "*musllinux*" CIBW_ARCHS: native CIBW_BUILD_FRONTEND: build CIBW_TEST_REQUIRES: pytest diff --git a/python/prophet/models.py b/python/prophet/models.py index a3d4f15..747b268 100644 --- a/python/prophet/models.py +++ b/python/prophet/models.py @@ -10,12 +10,17 @@ from typing import Tuple from collections import OrderedDict from enum import Enum from pathlib import Path +import os import pickle import pkg_resources +import platform import logging logger = logging.getLogger('prophet.models') +PLATFORM = "unix" +if platform.platform().startswith("Win"): + PLATFORM = "win" class IStanBackend(ABC): def __init__(self): @@ -66,8 +71,20 @@ class CmdStanPyBackend(IStanBackend): def get_type(): return StanBackendEnum.CMDSTANPY.name + def _add_tbb_to_path(self): + """Add the TBB library to $PATH on Windows only. Required for loading model binaries.""" + if PLATFORM == "win": + tbb_path = pkg_resources.resource_filename( + "prophet", + f"stan_model/cmdstan-{self.CMDSTAN_VERSION}/stan/lib/stan_math/lib/tbb" + ) + os.environ["PATH"] = ";".join( + list(OrderedDict.fromkeys([tbb_path] + os.environ.get("PATH", "").split(";"))) + ) + def load_model(self): import cmdstanpy + self._add_tbb_to_path() model_file = pkg_resources.resource_filename( 'prophet', 'stan_model/prophet_model.bin', diff --git a/python/setup.py b/python/setup.py index 7e999e6..de91ae2 100644 --- a/python/setup.py +++ b/python/setup.py @@ -137,13 +137,42 @@ def get_cmdstan_cache() -> str: def download_cmdstan(cache_dir: Path) -> None: """Ensure the cmdstan library exists in the cache directory.""" import cmdstanpy + from cmdstanpy.install_cmdstan import retrieve_version if cache_dir.is_dir(): print(f"Found existing cmdstan library at {cache_dir}") else: + print(f"Downloading cmdstan to {cache_dir}") cache_dir.parent.mkdir(parents=True, exist_ok=True) with cmdstanpy.utils.pushd(cache_dir.parent): - cmdstanpy.utils.retrieve_version(version=CMDSTAN_VERSION, progress=False) + retrieve_version(version=CMDSTAN_VERSION, progress=False) + + +def install_cmdstan_toolchain(): + """Install C++ compilers required to build stan models on Windows machines.""" + from cmdstanpy.install_cxx_toolchain import main as _install_cxx_toolchain + _install_cxx_toolchain() + + +def install_cmdstan_deps(cmdstan_dir: Path): + import cmdstanpy + + cmdstan_cache = get_cmdstan_cache() + download_cmdstan(cmdstan_cache) + if cmdstan_dir.is_dir(): + rmtree(cmdstan_dir) + copytree(cmdstan_cache, cmdstan_dir) + + if PLATFORM == "win": + try: + cmdstanpy.utils.cxx_toolchain_path() + except Exception: + install_cmdstan_toolchain() + + with cmdstanpy.utils.pushd(cmdstan_dir): + clean_all_cmdstan() + build_cmdstan() + cmdstanpy.set_cmdstan_path(str(cmdstan_dir)) def build_cmdstan_model(target_dir): @@ -157,19 +186,8 @@ def build_cmdstan_model(target_dir): target_dir: Directory to copy the compiled model executable and core cmdstan files to. """ import cmdstanpy - - cmdstan_cache = get_cmdstan_cache() - download_cmdstan(cmdstan_cache) - - cmdstan_dir = os.path.join(target_dir, f"cmdstan-{CMDSTAN_VERSION}") - if os.path.isdir(cmdstan_dir): - rmtree(cmdstan_dir) - copytree(cmdstan_cache, cmdstan_dir) - with cmdstanpy.utils.pushd(cmdstan_dir): - clean_all_cmdstan() - build_cmdstan() - cmdstanpy.set_cmdstan_path(cmdstan_dir) - + cmdstan_dir = (Path(target_dir) / f"cmdstan-{CMDSTAN_VERSION}").resolve() + install_cmdstan_deps(cmdstan_dir) model_name = "prophet.stan" target_name = "prophet_model.bin" sm = cmdstanpy.CmdStanModel(stan_file=os.path.join(MODEL_DIR, model_name)) @@ -205,7 +223,7 @@ def build_models(target_dir): print(f"Compiling {backend} model") if backend == "CMDSTANPY": build_cmdstan_model(target_dir) - elif backend == "PYSTAN": + elif backend == "PYSTAN" and PLATFORM != "win": build_pystan_model(target_dir)