From 39c619dbf7ba439260c42af764e9736ba57931c3 Mon Sep 17 00:00:00 2001 From: Ryan Nazareth Date: Mon, 9 Mar 2020 19:25:01 +0000 Subject: [PATCH] Running Cross-Validation with Multiprocessing Option (#1356) * Add multiprocessing Pool and create function for single cutoff forecast * add params to single forecast function * Add iterable for input params for pool * Add docstring for single cutoff forecast func * Add check for multiprocessing in test_cross_validation * check ofr is None and is True and add better description for multiprocess in docstring * Raise error if wrong args chosen and add test * fix conflicts * Change arg to True/False, model.kwargs * docstring units and few more fixes * change to iterator and add back tqdm to for loop * add option in diagnosics notebook about multiprocessing option * add extra test for checking calls to single forecast func --- notebooks/diagnostics.ipynb | 4 +- python/fbprophet/diagnostics.py | 106 ++++++++++++++------- python/fbprophet/tests/test_diagnostics.py | 79 +++++++++++---- 3 files changed, 136 insertions(+), 53 deletions(-) diff --git a/notebooks/diagnostics.ipynb b/notebooks/diagnostics.ipynb index 2abc447..148d6ec 100644 --- a/notebooks/diagnostics.ipynb +++ b/notebooks/diagnostics.ipynb @@ -263,6 +263,8 @@ "source": [ "In R, the argument `units` must be a type accepted by `as.difftime`, which is weeks or shorter. In Python, the string for `initial`, `period`, and `horizon` should be in the format used by Pandas Timedelta, which accepts units of days or shorter.\n", "\n", + "Cross-validation can also be run in multiprocessing mode in Python, by setting the `multiprocess` argument to `True`\n", + "\n", "The `performance_metrics` utility can be used to compute some useful statistics of the prediction performance (`yhat`, `yhat_lower`, and `yhat_upper` compared to `y`), as a function of the distance from the cutoff (how far into the future the prediction was). The statistics computed are mean squared error (MSE), root mean squared error (RMSE), mean absolute error (MAE), mean absolute percent error (MAPE), and coverage of the `yhat_lower` and `yhat_upper` estimates. These are computed on a rolling window of the predictions in `df_cv` after sorting by horizon (`ds` minus `cutoff`). By default 10% of the predictions will be included in each window, but this can be changed with the `rolling_window` argument." ] }, @@ -473,7 +475,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.0" } }, "nbformat": 4, diff --git a/python/fbprophet/diagnostics.py b/python/fbprophet/diagnostics.py index 93a6421..26c4eb0 100644 --- a/python/fbprophet/diagnostics.py +++ b/python/fbprophet/diagnostics.py @@ -13,6 +13,7 @@ from functools import reduce import numpy as np import pandas as pd +from multiprocessing import Pool logger = logging.getLogger('fbprophet') @@ -58,7 +59,7 @@ def generate_cutoffs(df, horizon, initial, period): return list(reversed(result)) -def cross_validation(model, horizon, period=None, initial=None): +def cross_validation(model, horizon, period=None, initial=None, multiprocess=False): """Cross-Validation for time series. Computes forecasts from historical cutoff points. Beginning from @@ -77,11 +78,15 @@ def cross_validation(model, horizon, period=None, initial=None): be done at every this period. If not provided, 0.5 * horizon is used. initial: string with pd.Timedelta compatible style. The first training period will begin here. If not provided, 3 * horizon is used. + multiprocess: True, False, Optional (defaults to False). If `True`, use the + `multiprocessing` module to distribute each task to a different processor + core. Returns ------- A pd.DataFrame with the forecast, actual value and cutoff. """ + df = model.history.copy().reset_index(drop=True) horizon = pd.Timedelta(horizon) # Set period @@ -106,43 +111,78 @@ def cross_validation(model, horizon, period=None, initial=None): if model.uncertainty_samples: predict_columns.extend(['yhat_lower', 'yhat_upper']) - cutoffs = generate_cutoffs(df, horizon, initial, period) predicts = [] - for cutoff in tqdm(cutoffs): - # Generate new object with copying fitting options - m = prophet_copy(model, cutoff) - # Train model - history_c = df[df['ds'] <= cutoff] - if history_c.shape[0] < 2: - raise Exception( - 'Less than two datapoints before cutoff. ' - 'Increase initial window.' - ) - m.fit(history_c, **model.fit_kwargs) - # Calculate yhat - index_predicted = (df['ds'] > cutoff) & (df['ds'] <= cutoff + horizon) - # Get the columns for the future dataframe - columns = ['ds'] - if m.growth == 'logistic': - columns.append('cap') - if m.logistic_floor: - columns.append('floor') - columns.extend(m.extra_regressors.keys()) - columns.extend([ - props['condition_name'] - for props in m.seasonalities.values() - if props['condition_name'] is not None]) - yhat = m.predict(df[index_predicted][columns]) - # Merge yhat(predicts), y(df, original data) and cutoff - predicts.append(pd.concat([ - yhat[predict_columns], - df[index_predicted][['y']].reset_index(drop=True), - pd.DataFrame({'cutoff': [cutoff] * len(yhat)}) - ], axis=1)) + cutoffs = generate_cutoffs(df, horizon, initial, period) + + if multiprocess is True: + with Pool() as pool: + logger.info('Running cross validation in multiprocessing mode') + input_df = ((df, model, cutoff, horizon, predict_columns) for cutoff in tqdm(cutoffs)) + predicts = pool.starmap(single_cutoff_forecast, input_df) + else: + if multiprocess is False: + for cutoff in tqdm(cutoffs): + predicts.append(single_cutoff_forecast(df, model, cutoff, horizon, predict_columns)) # Combine all predicted pd.DataFrame into one pd.DataFrame return pd.concat(predicts, axis=0).reset_index(drop=True) + +def single_cutoff_forecast(df, model, cutoff, horizon, predict_columns): + """Forecast for single cutoff. Used in cross validation function + when evaluating for multiple cutoffs either sequentially or in parallel . + + Parameters + ---------- + df: pd.DataFrame. + DataFrame with history to be used for single + cutoff forecast. + model: Prophet model object. + cutoff: pd.Timestamp cutoff date. + Simulated Forecast will start from this date. + horizon: pd.Timedelta forecast horizon. + predict_columns: List of strings e.g. ['ds', 'yhat']. + Columns with date and forecast to be returned in output. + + Returns + ------- + A pd.DataFrame with the forecast, actual value and cutoff. + + """ + + # Generate new object with copying fitting options + m = prophet_copy(model, cutoff) + # Train model + history_c = df[df['ds'] <= cutoff] + if history_c.shape[0] < 2: + raise Exception( + 'Less than two datapoints before cutoff. ' + 'Increase initial window.' + ) + m.fit(history_c, **model.fit_kwargs) + # Calculate yhat + index_predicted = (df['ds'] > cutoff) & (df['ds'] <= cutoff + horizon) + # Get the columns for the future dataframe + columns = ['ds'] + if m.growth == 'logistic': + columns.append('cap') + if m.logistic_floor: + columns.append('floor') + columns.extend(m.extra_regressors.keys()) + columns.extend([ + props['condition_name'] + for props in m.seasonalities.values() + if props['condition_name'] is not None]) + yhat = m.predict(df[index_predicted][columns]) + # Merge yhat(predicts), y(df, original data) and cutoff + + return pd.concat([ + yhat[predict_columns], + df[index_predicted][['y']].reset_index(drop=True), + pd.DataFrame({'cutoff': [cutoff] * len(yhat)}) + ], axis=1) + + def prophet_copy(m, cutoff=None): """Copy Prophet object diff --git a/python/fbprophet/tests/test_diagnostics.py b/python/fbprophet/tests/test_diagnostics.py index d50d859..b374e8d 100644 --- a/python/fbprophet/tests/test_diagnostics.py +++ b/python/fbprophet/tests/test_diagnostics.py @@ -11,9 +11,11 @@ from __future__ import unicode_literals import itertools import os from unittest import TestCase +from unittest.mock import patch import numpy as np import pandas as pd +import datetime from fbprophet import Prophet from fbprophet import diagnostics @@ -38,25 +40,64 @@ class TestDiagnostics(TestCase): horizon = pd.Timedelta('4 days') period = pd.Timedelta('10 days') initial = pd.Timedelta('115 days') - df_cv = diagnostics.cross_validation( - m, horizon='4 days', period='10 days', initial='115 days') - self.assertEqual(len(np.unique(df_cv['cutoff'])), 3) - self.assertEqual(max(df_cv['ds'] - df_cv['cutoff']), horizon) - self.assertTrue(min(df_cv['cutoff']) >= min(self.__df['ds']) + initial) - dc = df_cv['cutoff'].diff() - dc = dc[dc > pd.Timedelta(0)].min() - self.assertTrue(dc >= period) - self.assertTrue((df_cv['cutoff'] < df_cv['ds']).all()) - # Each y in df_cv and self.__df with same ds should be equal - df_merged = pd.merge(df_cv, self.__df, 'left', on='ds') - self.assertAlmostEqual( - np.sum((df_merged['y_x'] - df_merged['y_y']) ** 2), 0.0) - df_cv = diagnostics.cross_validation( - m, horizon='4 days', period='10 days', initial='135 days') - self.assertEqual(len(np.unique(df_cv['cutoff'])), 1) - with self.assertRaises(ValueError): - diagnostics.cross_validation( - m, horizon='10 days', period='10 days', initial='140 days') + # Run for both cases of multiprocess on or off + for multiprocess in [False, True]: + df_cv = diagnostics.cross_validation( + m, horizon='4 days', period='10 days', initial='115 days', + multiprocess=multiprocess) + self.assertEqual(len(np.unique(df_cv['cutoff'])), 3) + self.assertEqual(max(df_cv['ds'] - df_cv['cutoff']), horizon) + self.assertTrue(min(df_cv['cutoff']) >= min(self.__df['ds']) + initial) + dc = df_cv['cutoff'].diff() + dc = dc[dc > pd.Timedelta(0)].min() + self.assertTrue(dc >= period) + self.assertTrue((df_cv['cutoff'] < df_cv['ds']).all()) + # Each y in df_cv and self.__df with same ds should be equal + df_merged = pd.merge(df_cv, self.__df, 'left', on='ds') + self.assertAlmostEqual( + np.sum((df_merged['y_x'] - df_merged['y_y']) ** 2), 0.0) + df_cv = diagnostics.cross_validation( + m, horizon='4 days', period='10 days', initial='135 days') + self.assertEqual(len(np.unique(df_cv['cutoff'])), 1) + with self.assertRaises(ValueError): + diagnostics.cross_validation( + m, horizon='10 days', period='10 days', initial='140 days') + + def test_invalid_args_multiprocess(self): + m = Prophet() + m.fit(self.__df) + # Calculate the number of cutoff points(k) + horizon = pd.Timedelta('4 days') + period = pd.Timedelta('10 days') + initial = pd.Timedelta('115 days') + # Run for both cases of multiprocess on or off + for multiprocess in [1, 'yes']: + with self.assertRaises(ValueError): + df_cv = diagnostics.cross_validation( + m, horizon='4 days', period='10 days', initial='115 days', + multiprocess=multiprocess) + + + def test_check_single_cutoff_forecast_func_calls(self): + m = Prophet() + m.fit(self.__df) + mock_predict = pd.DataFrame({'ds':pd.date_range(start='2012-09-17', periods=3), + 'yhat':np.arange(16, 19), + 'yhat_lower':np.arange(15, 18), + 'yhat_upper': np.arange(17, 20), + 'y': np.arange(16.5, 19.5), + 'cutoff': [datetime.date(2012, 9, 15)]*3}) + + # cross validation with 3 and 7 forecasts + for args, forecasts in ((['4 days', '10 days', '115 days'], 3), + (['4 days', '4 days', '115 days'], 7)): + with patch('fbprophet.diagnostics.single_cutoff_forecast') as mock_func: + mock_func.return_value = mock_predict + df_cv = diagnostics.cross_validation(m, *args) + # check single forecast function called expected number of times + self.assertEqual(diagnostics.single_cutoff_forecast.call_count, + forecasts) + def test_cross_validation_logistic(self): df = self.__df.copy()