Running Cross-Validation with Multiprocessing Option (#1356)

* Add multiprocessing Pool and create function for single cutoff forecast

* add params to single forecast function

* Add iterable for input params for pool

* Add docstring for single cutoff forecast func

* Add check for multiprocessing in test_cross_validation

* check ofr is None and is True and add better description for multiprocess in docstring

* Raise error if wrong args chosen and add test

* fix conflicts

* Change arg to True/False, model.kwargs

* docstring units and few more fixes

* change to iterator and add back tqdm to for loop

* add option in diagnosics notebook about multiprocessing option

* add extra test for checking calls to single forecast func
This commit is contained in:
Ryan Nazareth 2020-03-09 19:25:01 +00:00 committed by GitHub
parent fd61fcbeeb
commit 39c619dbf7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 136 additions and 53 deletions

View file

@ -263,6 +263,8 @@
"source": [
"In R, the argument `units` must be a type accepted by `as.difftime`, which is weeks or shorter. In Python, the string for `initial`, `period`, and `horizon` should be in the format used by Pandas Timedelta, which accepts units of days or shorter.\n",
"\n",
"Cross-validation can also be run in multiprocessing mode in Python, by setting the `multiprocess` argument to `True`\n",
"\n",
"The `performance_metrics` utility can be used to compute some useful statistics of the prediction performance (`yhat`, `yhat_lower`, and `yhat_upper` compared to `y`), as a function of the distance from the cutoff (how far into the future the prediction was). The statistics computed are mean squared error (MSE), root mean squared error (RMSE), mean absolute error (MAE), mean absolute percent error (MAPE), and coverage of the `yhat_lower` and `yhat_upper` estimates. These are computed on a rolling window of the predictions in `df_cv` after sorting by horizon (`ds` minus `cutoff`). By default 10% of the predictions will be included in each window, but this can be changed with the `rolling_window` argument."
]
},
@ -473,7 +475,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
"version": "3.7.0"
}
},
"nbformat": 4,

View file

@ -13,6 +13,7 @@ from functools import reduce
import numpy as np
import pandas as pd
from multiprocessing import Pool
logger = logging.getLogger('fbprophet')
@ -58,7 +59,7 @@ def generate_cutoffs(df, horizon, initial, period):
return list(reversed(result))
def cross_validation(model, horizon, period=None, initial=None):
def cross_validation(model, horizon, period=None, initial=None, multiprocess=False):
"""Cross-Validation for time series.
Computes forecasts from historical cutoff points. Beginning from
@ -77,11 +78,15 @@ def cross_validation(model, horizon, period=None, initial=None):
be done at every this period. If not provided, 0.5 * horizon is used.
initial: string with pd.Timedelta compatible style. The first training
period will begin here. If not provided, 3 * horizon is used.
multiprocess: True, False, Optional (defaults to False). If `True`, use the
`multiprocessing` module to distribute each task to a different processor
core.
Returns
-------
A pd.DataFrame with the forecast, actual value and cutoff.
"""
df = model.history.copy().reset_index(drop=True)
horizon = pd.Timedelta(horizon)
# Set period
@ -106,43 +111,78 @@ def cross_validation(model, horizon, period=None, initial=None):
if model.uncertainty_samples:
predict_columns.extend(['yhat_lower', 'yhat_upper'])
cutoffs = generate_cutoffs(df, horizon, initial, period)
predicts = []
for cutoff in tqdm(cutoffs):
# Generate new object with copying fitting options
m = prophet_copy(model, cutoff)
# Train model
history_c = df[df['ds'] <= cutoff]
if history_c.shape[0] < 2:
raise Exception(
'Less than two datapoints before cutoff. '
'Increase initial window.'
)
m.fit(history_c, **model.fit_kwargs)
# Calculate yhat
index_predicted = (df['ds'] > cutoff) & (df['ds'] <= cutoff + horizon)
# Get the columns for the future dataframe
columns = ['ds']
if m.growth == 'logistic':
columns.append('cap')
if m.logistic_floor:
columns.append('floor')
columns.extend(m.extra_regressors.keys())
columns.extend([
props['condition_name']
for props in m.seasonalities.values()
if props['condition_name'] is not None])
yhat = m.predict(df[index_predicted][columns])
# Merge yhat(predicts), y(df, original data) and cutoff
predicts.append(pd.concat([
yhat[predict_columns],
df[index_predicted][['y']].reset_index(drop=True),
pd.DataFrame({'cutoff': [cutoff] * len(yhat)})
], axis=1))
cutoffs = generate_cutoffs(df, horizon, initial, period)
if multiprocess is True:
with Pool() as pool:
logger.info('Running cross validation in multiprocessing mode')
input_df = ((df, model, cutoff, horizon, predict_columns) for cutoff in tqdm(cutoffs))
predicts = pool.starmap(single_cutoff_forecast, input_df)
else:
if multiprocess is False:
for cutoff in tqdm(cutoffs):
predicts.append(single_cutoff_forecast(df, model, cutoff, horizon, predict_columns))
# Combine all predicted pd.DataFrame into one pd.DataFrame
return pd.concat(predicts, axis=0).reset_index(drop=True)
def single_cutoff_forecast(df, model, cutoff, horizon, predict_columns):
"""Forecast for single cutoff. Used in cross validation function
when evaluating for multiple cutoffs either sequentially or in parallel .
Parameters
----------
df: pd.DataFrame.
DataFrame with history to be used for single
cutoff forecast.
model: Prophet model object.
cutoff: pd.Timestamp cutoff date.
Simulated Forecast will start from this date.
horizon: pd.Timedelta forecast horizon.
predict_columns: List of strings e.g. ['ds', 'yhat'].
Columns with date and forecast to be returned in output.
Returns
-------
A pd.DataFrame with the forecast, actual value and cutoff.
"""
# Generate new object with copying fitting options
m = prophet_copy(model, cutoff)
# Train model
history_c = df[df['ds'] <= cutoff]
if history_c.shape[0] < 2:
raise Exception(
'Less than two datapoints before cutoff. '
'Increase initial window.'
)
m.fit(history_c, **model.fit_kwargs)
# Calculate yhat
index_predicted = (df['ds'] > cutoff) & (df['ds'] <= cutoff + horizon)
# Get the columns for the future dataframe
columns = ['ds']
if m.growth == 'logistic':
columns.append('cap')
if m.logistic_floor:
columns.append('floor')
columns.extend(m.extra_regressors.keys())
columns.extend([
props['condition_name']
for props in m.seasonalities.values()
if props['condition_name'] is not None])
yhat = m.predict(df[index_predicted][columns])
# Merge yhat(predicts), y(df, original data) and cutoff
return pd.concat([
yhat[predict_columns],
df[index_predicted][['y']].reset_index(drop=True),
pd.DataFrame({'cutoff': [cutoff] * len(yhat)})
], axis=1)
def prophet_copy(m, cutoff=None):
"""Copy Prophet object

View file

@ -11,9 +11,11 @@ from __future__ import unicode_literals
import itertools
import os
from unittest import TestCase
from unittest.mock import patch
import numpy as np
import pandas as pd
import datetime
from fbprophet import Prophet
from fbprophet import diagnostics
@ -38,25 +40,64 @@ class TestDiagnostics(TestCase):
horizon = pd.Timedelta('4 days')
period = pd.Timedelta('10 days')
initial = pd.Timedelta('115 days')
df_cv = diagnostics.cross_validation(
m, horizon='4 days', period='10 days', initial='115 days')
self.assertEqual(len(np.unique(df_cv['cutoff'])), 3)
self.assertEqual(max(df_cv['ds'] - df_cv['cutoff']), horizon)
self.assertTrue(min(df_cv['cutoff']) >= min(self.__df['ds']) + initial)
dc = df_cv['cutoff'].diff()
dc = dc[dc > pd.Timedelta(0)].min()
self.assertTrue(dc >= period)
self.assertTrue((df_cv['cutoff'] < df_cv['ds']).all())
# Each y in df_cv and self.__df with same ds should be equal
df_merged = pd.merge(df_cv, self.__df, 'left', on='ds')
self.assertAlmostEqual(
np.sum((df_merged['y_x'] - df_merged['y_y']) ** 2), 0.0)
df_cv = diagnostics.cross_validation(
m, horizon='4 days', period='10 days', initial='135 days')
self.assertEqual(len(np.unique(df_cv['cutoff'])), 1)
with self.assertRaises(ValueError):
diagnostics.cross_validation(
m, horizon='10 days', period='10 days', initial='140 days')
# Run for both cases of multiprocess on or off
for multiprocess in [False, True]:
df_cv = diagnostics.cross_validation(
m, horizon='4 days', period='10 days', initial='115 days',
multiprocess=multiprocess)
self.assertEqual(len(np.unique(df_cv['cutoff'])), 3)
self.assertEqual(max(df_cv['ds'] - df_cv['cutoff']), horizon)
self.assertTrue(min(df_cv['cutoff']) >= min(self.__df['ds']) + initial)
dc = df_cv['cutoff'].diff()
dc = dc[dc > pd.Timedelta(0)].min()
self.assertTrue(dc >= period)
self.assertTrue((df_cv['cutoff'] < df_cv['ds']).all())
# Each y in df_cv and self.__df with same ds should be equal
df_merged = pd.merge(df_cv, self.__df, 'left', on='ds')
self.assertAlmostEqual(
np.sum((df_merged['y_x'] - df_merged['y_y']) ** 2), 0.0)
df_cv = diagnostics.cross_validation(
m, horizon='4 days', period='10 days', initial='135 days')
self.assertEqual(len(np.unique(df_cv['cutoff'])), 1)
with self.assertRaises(ValueError):
diagnostics.cross_validation(
m, horizon='10 days', period='10 days', initial='140 days')
def test_invalid_args_multiprocess(self):
m = Prophet()
m.fit(self.__df)
# Calculate the number of cutoff points(k)
horizon = pd.Timedelta('4 days')
period = pd.Timedelta('10 days')
initial = pd.Timedelta('115 days')
# Run for both cases of multiprocess on or off
for multiprocess in [1, 'yes']:
with self.assertRaises(ValueError):
df_cv = diagnostics.cross_validation(
m, horizon='4 days', period='10 days', initial='115 days',
multiprocess=multiprocess)
def test_check_single_cutoff_forecast_func_calls(self):
m = Prophet()
m.fit(self.__df)
mock_predict = pd.DataFrame({'ds':pd.date_range(start='2012-09-17', periods=3),
'yhat':np.arange(16, 19),
'yhat_lower':np.arange(15, 18),
'yhat_upper': np.arange(17, 20),
'y': np.arange(16.5, 19.5),
'cutoff': [datetime.date(2012, 9, 15)]*3})
# cross validation with 3 and 7 forecasts
for args, forecasts in ((['4 days', '10 days', '115 days'], 3),
(['4 days', '4 days', '115 days'], 7)):
with patch('fbprophet.diagnostics.single_cutoff_forecast') as mock_func:
mock_func.return_value = mock_predict
df_cv = diagnostics.cross_validation(m, *args)
# check single forecast function called expected number of times
self.assertEqual(diagnostics.single_cutoff_forecast.call_count,
forecasts)
def test_cross_validation_logistic(self):
df = self.__df.copy()